From d2d38d6fe3931c132ccfcff31da1696946181b38 Mon Sep 17 00:00:00 2001 From: ZTWHHH Date: Wed, 10 Jun 2026 15:24:15 +0000 Subject: [PATCH 1/6] Add test scripts and results for 5 new model series Test scripts and result.json files for the model series introduced in the companion main-branch PR: - kimi-vl - minicpm-v-4d5 - glm-4d1v - ovis2 - aria Each series has two scripts (text, single-image-start) and corresponding work_dirs//-/result.json under the same naming convention used elsewhere in this branch. Co-Authored-By: Claude Opus 4.7 --- scripts/aria/aria-single-image-start.sh | 15 ++++++ scripts/aria/aria-text.sh | 14 ++++++ .../glm-4d1v/glm-4d1v-single-image-start.sh | 15 ++++++ scripts/glm-4d1v/glm-4d1v-text.sh | 14 ++++++ scripts/kimi-vl/kimi-vl-single-image-start.sh | 15 ++++++ scripts/kimi-vl/kimi-vl-text.sh | 14 ++++++ .../minicpm-v-4d5-single-image-start.sh | 15 ++++++ scripts/minicpm-v-4d5/minicpm-v-4d5-text.sh | 14 ++++++ scripts/ovis2/ovis2-single-image-start.sh | 15 ++++++ scripts/ovis2/ovis2-text.sh | 14 ++++++ .../aria/Aria-single-image-start/result.json | 48 +++++++++++++++++++ work_dirs/aria/Aria-text/result.json | 44 +++++++++++++++++ .../result.json | 48 +++++++++++++++++++ .../GLM-4.1V-9B-Thinking-text/result.json | 44 +++++++++++++++++ .../result.json | 48 +++++++++++++++++++ .../Kimi-VL-A3B-Instruct-text/result.json | 44 +++++++++++++++++ .../result.json | 48 +++++++++++++++++++ .../MiniCPM-V-4_5-text/result.json | 44 +++++++++++++++++ .../Ovis2-8B-single-image-start/result.json | 48 +++++++++++++++++++ work_dirs/ovis2/Ovis2-8B-text/result.json | 44 +++++++++++++++++ 20 files changed, 605 insertions(+) create mode 100644 scripts/aria/aria-single-image-start.sh create mode 100644 scripts/aria/aria-text.sh create mode 100644 scripts/glm-4d1v/glm-4d1v-single-image-start.sh create mode 100644 scripts/glm-4d1v/glm-4d1v-text.sh create mode 100644 scripts/kimi-vl/kimi-vl-single-image-start.sh create mode 100644 scripts/kimi-vl/kimi-vl-text.sh create mode 100644 scripts/minicpm-v-4d5/minicpm-v-4d5-single-image-start.sh create mode 100644 scripts/minicpm-v-4d5/minicpm-v-4d5-text.sh create mode 100644 scripts/ovis2/ovis2-single-image-start.sh create mode 100644 scripts/ovis2/ovis2-text.sh create mode 100644 work_dirs/aria/Aria-single-image-start/result.json create mode 100644 work_dirs/aria/Aria-text/result.json create mode 100644 work_dirs/glm-4d1v/GLM-4.1V-9B-Thinking-single-image-start/result.json create mode 100644 work_dirs/glm-4d1v/GLM-4.1V-9B-Thinking-text/result.json create mode 100644 work_dirs/kimi-vl/Kimi-VL-A3B-Instruct-single-image-start/result.json create mode 100644 work_dirs/kimi-vl/Kimi-VL-A3B-Instruct-text/result.json create mode 100644 work_dirs/minicpm-v-4d5/MiniCPM-V-4_5-single-image-start/result.json create mode 100644 work_dirs/minicpm-v-4d5/MiniCPM-V-4_5-text/result.json create mode 100644 work_dirs/ovis2/Ovis2-8B-single-image-start/result.json create mode 100644 work_dirs/ovis2/Ovis2-8B-text/result.json diff --git a/scripts/aria/aria-single-image-start.sh b/scripts/aria/aria-single-image-start.sh new file mode 100644 index 00000000..ff5b83a0 --- /dev/null +++ b/scripts/aria/aria-single-image-start.sh @@ -0,0 +1,15 @@ +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +MMEVAL_DIR="$SCRIPT_DIR/../../../simple-mmeval-model-dev" +RESULT_DIR="$SCRIPT_DIR/../.." +export PYTHONPATH="$MMEVAL_DIR:$PYTHONPATH" +cd "$MMEVAL_DIR" + +python $MMEVAL_DIR/mmeval/run.py \ + --infile $MMEVAL_DIR/tests/samples/single_image_start.json \ + --dataset local@json \ + --out_dir $RESULT_DIR/work_dirs/aria/Aria-single-image-start \ + --img_dir $MMEVAL_DIR/tests/media/448 \ + --model_name_or_path rhymes-ai/Aria \ + --gpu_per_parallel 1 \ + --parallel_per_task 1 \ + --max_new_tokens 512 diff --git a/scripts/aria/aria-text.sh b/scripts/aria/aria-text.sh new file mode 100644 index 00000000..1c96ce52 --- /dev/null +++ b/scripts/aria/aria-text.sh @@ -0,0 +1,14 @@ +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +MMEVAL_DIR="$SCRIPT_DIR/../../../simple-mmeval-model-dev" +RESULT_DIR="$SCRIPT_DIR/../.." +export PYTHONPATH="$MMEVAL_DIR:$PYTHONPATH" +cd "$MMEVAL_DIR" + +python $MMEVAL_DIR/mmeval/run.py \ + --infile $MMEVAL_DIR/tests/samples/no_media.json \ + --dataset local@json \ + --out_dir $RESULT_DIR/work_dirs/aria/Aria-text \ + --model_name_or_path rhymes-ai/Aria \ + --gpu_per_parallel 1 \ + --parallel_per_task 1 \ + --max_new_tokens 512 diff --git a/scripts/glm-4d1v/glm-4d1v-single-image-start.sh b/scripts/glm-4d1v/glm-4d1v-single-image-start.sh new file mode 100644 index 00000000..baeee74a --- /dev/null +++ b/scripts/glm-4d1v/glm-4d1v-single-image-start.sh @@ -0,0 +1,15 @@ +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +MMEVAL_DIR="$SCRIPT_DIR/../../../simple-mmeval-model-dev" +RESULT_DIR="$SCRIPT_DIR/../.." +export PYTHONPATH="$MMEVAL_DIR:$PYTHONPATH" +cd "$MMEVAL_DIR" + +python $MMEVAL_DIR/mmeval/run.py \ + --infile $MMEVAL_DIR/tests/samples/single_image_start.json \ + --dataset local@json \ + --out_dir $RESULT_DIR/work_dirs/glm-4d1v/GLM-4.1V-9B-Thinking-single-image-start \ + --img_dir $MMEVAL_DIR/tests/media/448 \ + --model_name_or_path THUDM/GLM-4.1V-9B-Thinking \ + --gpu_per_parallel 1 \ + --parallel_per_task 1 \ + --max_new_tokens 512 diff --git a/scripts/glm-4d1v/glm-4d1v-text.sh b/scripts/glm-4d1v/glm-4d1v-text.sh new file mode 100644 index 00000000..a2f58e02 --- /dev/null +++ b/scripts/glm-4d1v/glm-4d1v-text.sh @@ -0,0 +1,14 @@ +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +MMEVAL_DIR="$SCRIPT_DIR/../../../simple-mmeval-model-dev" +RESULT_DIR="$SCRIPT_DIR/../.." +export PYTHONPATH="$MMEVAL_DIR:$PYTHONPATH" +cd "$MMEVAL_DIR" + +python $MMEVAL_DIR/mmeval/run.py \ + --infile $MMEVAL_DIR/tests/samples/no_media.json \ + --dataset local@json \ + --out_dir $RESULT_DIR/work_dirs/glm-4d1v/GLM-4.1V-9B-Thinking-text \ + --model_name_or_path THUDM/GLM-4.1V-9B-Thinking \ + --gpu_per_parallel 1 \ + --parallel_per_task 1 \ + --max_new_tokens 512 diff --git a/scripts/kimi-vl/kimi-vl-single-image-start.sh b/scripts/kimi-vl/kimi-vl-single-image-start.sh new file mode 100644 index 00000000..9caafbb7 --- /dev/null +++ b/scripts/kimi-vl/kimi-vl-single-image-start.sh @@ -0,0 +1,15 @@ +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +MMEVAL_DIR="$SCRIPT_DIR/../../../simple-mmeval-model-dev" +RESULT_DIR="$SCRIPT_DIR/../.." +export PYTHONPATH="$MMEVAL_DIR:$PYTHONPATH" +cd "$MMEVAL_DIR" + +python $MMEVAL_DIR/mmeval/run.py \ + --infile $MMEVAL_DIR/tests/samples/single_image_start.json \ + --dataset local@json \ + --out_dir $RESULT_DIR/work_dirs/kimi-vl/Kimi-VL-A3B-Instruct-single-image-start \ + --img_dir $MMEVAL_DIR/tests/media/448 \ + --model_name_or_path moonshotai/Kimi-VL-A3B-Instruct \ + --gpu_per_parallel 1 \ + --parallel_per_task 1 \ + --max_new_tokens 512 diff --git a/scripts/kimi-vl/kimi-vl-text.sh b/scripts/kimi-vl/kimi-vl-text.sh new file mode 100644 index 00000000..67d78d23 --- /dev/null +++ b/scripts/kimi-vl/kimi-vl-text.sh @@ -0,0 +1,14 @@ +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +MMEVAL_DIR="$SCRIPT_DIR/../../../simple-mmeval-model-dev" +RESULT_DIR="$SCRIPT_DIR/../.." +export PYTHONPATH="$MMEVAL_DIR:$PYTHONPATH" +cd "$MMEVAL_DIR" + +python $MMEVAL_DIR/mmeval/run.py \ + --infile $MMEVAL_DIR/tests/samples/no_media.json \ + --dataset local@json \ + --out_dir $RESULT_DIR/work_dirs/kimi-vl/Kimi-VL-A3B-Instruct-text \ + --model_name_or_path moonshotai/Kimi-VL-A3B-Instruct \ + --gpu_per_parallel 1 \ + --parallel_per_task 1 \ + --max_new_tokens 512 diff --git a/scripts/minicpm-v-4d5/minicpm-v-4d5-single-image-start.sh b/scripts/minicpm-v-4d5/minicpm-v-4d5-single-image-start.sh new file mode 100644 index 00000000..71078ca6 --- /dev/null +++ b/scripts/minicpm-v-4d5/minicpm-v-4d5-single-image-start.sh @@ -0,0 +1,15 @@ +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +MMEVAL_DIR="$SCRIPT_DIR/../../../simple-mmeval-model-dev" +RESULT_DIR="$SCRIPT_DIR/../.." +export PYTHONPATH="$MMEVAL_DIR:$PYTHONPATH" +cd "$MMEVAL_DIR" + +python $MMEVAL_DIR/mmeval/run.py \ + --infile $MMEVAL_DIR/tests/samples/single_image_start.json \ + --dataset local@json \ + --out_dir $RESULT_DIR/work_dirs/minicpm-v-4d5/MiniCPM-V-4_5-single-image-start \ + --img_dir $MMEVAL_DIR/tests/media/448 \ + --model_name_or_path openbmb/MiniCPM-V-4_5 \ + --gpu_per_parallel 1 \ + --parallel_per_task 1 \ + --max_new_tokens 512 diff --git a/scripts/minicpm-v-4d5/minicpm-v-4d5-text.sh b/scripts/minicpm-v-4d5/minicpm-v-4d5-text.sh new file mode 100644 index 00000000..ba465450 --- /dev/null +++ b/scripts/minicpm-v-4d5/minicpm-v-4d5-text.sh @@ -0,0 +1,14 @@ +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +MMEVAL_DIR="$SCRIPT_DIR/../../../simple-mmeval-model-dev" +RESULT_DIR="$SCRIPT_DIR/../.." +export PYTHONPATH="$MMEVAL_DIR:$PYTHONPATH" +cd "$MMEVAL_DIR" + +python $MMEVAL_DIR/mmeval/run.py \ + --infile $MMEVAL_DIR/tests/samples/no_media.json \ + --dataset local@json \ + --out_dir $RESULT_DIR/work_dirs/minicpm-v-4d5/MiniCPM-V-4_5-text \ + --model_name_or_path openbmb/MiniCPM-V-4_5 \ + --gpu_per_parallel 1 \ + --parallel_per_task 1 \ + --max_new_tokens 512 diff --git a/scripts/ovis2/ovis2-single-image-start.sh b/scripts/ovis2/ovis2-single-image-start.sh new file mode 100644 index 00000000..d720bb39 --- /dev/null +++ b/scripts/ovis2/ovis2-single-image-start.sh @@ -0,0 +1,15 @@ +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +MMEVAL_DIR="$SCRIPT_DIR/../../../simple-mmeval-model-dev" +RESULT_DIR="$SCRIPT_DIR/../.." +export PYTHONPATH="$MMEVAL_DIR:$PYTHONPATH" +cd "$MMEVAL_DIR" + +python $MMEVAL_DIR/mmeval/run.py \ + --infile $MMEVAL_DIR/tests/samples/single_image_start.json \ + --dataset local@json \ + --out_dir $RESULT_DIR/work_dirs/ovis2/Ovis2-8B-single-image-start \ + --img_dir $MMEVAL_DIR/tests/media/448 \ + --model_name_or_path AIDC-AI/Ovis2-8B \ + --gpu_per_parallel 1 \ + --parallel_per_task 1 \ + --max_new_tokens 512 diff --git a/scripts/ovis2/ovis2-text.sh b/scripts/ovis2/ovis2-text.sh new file mode 100644 index 00000000..cb2190a6 --- /dev/null +++ b/scripts/ovis2/ovis2-text.sh @@ -0,0 +1,14 @@ +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +MMEVAL_DIR="$SCRIPT_DIR/../../../simple-mmeval-model-dev" +RESULT_DIR="$SCRIPT_DIR/../.." +export PYTHONPATH="$MMEVAL_DIR:$PYTHONPATH" +cd "$MMEVAL_DIR" + +python $MMEVAL_DIR/mmeval/run.py \ + --infile $MMEVAL_DIR/tests/samples/no_media.json \ + --dataset local@json \ + --out_dir $RESULT_DIR/work_dirs/ovis2/Ovis2-8B-text \ + --model_name_or_path AIDC-AI/Ovis2-8B \ + --gpu_per_parallel 1 \ + --parallel_per_task 1 \ + --max_new_tokens 512 diff --git a/work_dirs/aria/Aria-single-image-start/result.json b/work_dirs/aria/Aria-single-image-start/result.json new file mode 100644 index 00000000..55545349 --- /dev/null +++ b/work_dirs/aria/Aria-single-image-start/result.json @@ -0,0 +1,48 @@ +[ + { + "id": 0, + "media": [ + "truck.png" + ], + "messages": [ + { + "role": "user", + "question": " Please provide a detailed description of the contents shown in the image.", + "answer": "", + "options": {}, + "choices": [], + "prompt": " Please provide a detailed description of the contents shown in the image.", + "hint": "" + }, + { + "role": "assistant", + "response": "The image shows a powerful GMC pickup truck in motion, driving off-road. It's a four-door model with a sleek, modern design. The truck is a metallic shade of gray, with black rims and side mirrors. The terrain beneath it is rough, dry dirt, and it is kicking" + } + ], + "comment": "Test 'single image start' modality.", + "eval-id": 0 + }, + { + "id": 1, + "media": [ + "boat.png" + ], + "messages": [ + { + "role": "user", + "question": " Please provide a detailed description of the contents shown in the image.", + "answer": "", + "options": {}, + "choices": [], + "prompt": " Please provide a detailed description of the contents shown in the image.", + "hint": "" + }, + { + "role": "assistant", + "response": "The image shows a sleek, white motor yacht cruising through calm seas under a clear blue sky. The yacht features a modern design with a low profile and a streamlined hull. There is a prominent bow wave created as the boat cuts through the water, indicating it is moving at a high speed." + } + ], + "comment": "Test 'single image start' modality.", + "eval-id": 1 + } +] \ No newline at end of file diff --git a/work_dirs/aria/Aria-text/result.json b/work_dirs/aria/Aria-text/result.json new file mode 100644 index 00000000..616d6528 --- /dev/null +++ b/work_dirs/aria/Aria-text/result.json @@ -0,0 +1,44 @@ +[ + { + "id": 0, + "media": [], + "messages": [ + { + "role": "user", + "question": "What is the result of 1 plus 1?", + "answer": "", + "options": {}, + "choices": [], + "prompt": "What is the result of 1 plus 1?", + "hint": "" + }, + { + "role": "assistant", + "response": "The result of $1 + 1$ is $2$.<|im_end|>" + } + ], + "comment": "Test 'no media' modality.", + "eval-id": 0 + }, + { + "id": 1, + "media": [], + "messages": [ + { + "role": "user", + "question": "What is the capital city of China?", + "answer": "", + "options": {}, + "choices": [], + "prompt": "What is the capital city of China?", + "hint": "" + }, + { + "role": "assistant", + "response": "The capital city of China is Beijing.<|im_end|>" + } + ], + "comment": "Test 'no media' modality.", + "eval-id": 1 + } +] \ No newline at end of file diff --git a/work_dirs/glm-4d1v/GLM-4.1V-9B-Thinking-single-image-start/result.json b/work_dirs/glm-4d1v/GLM-4.1V-9B-Thinking-single-image-start/result.json new file mode 100644 index 00000000..544174c4 --- /dev/null +++ b/work_dirs/glm-4d1v/GLM-4.1V-9B-Thinking-single-image-start/result.json @@ -0,0 +1,48 @@ +[ + { + "id": 0, + "media": [ + "truck.png" + ], + "messages": [ + { + "role": "user", + "question": " Please provide a detailed description of the contents shown in the image.", + "answer": "", + "options": {}, + "choices": [], + "prompt": " Please provide a detailed description of the contents shown in the image.", + "hint": "" + }, + { + "role": "assistant", + "response": "Got it, let's start by describing the image. The main subject is a GMC pickup truck, specifically a modern model, probably the 2024 GMC Hummer EV or a similar model, but wait, no, the design looks like the GMC Sierra or maybe the Hummer pickup? Wait, the image shows" + } + ], + "comment": "Test 'single image start' modality.", + "eval-id": 0 + }, + { + "id": 1, + "media": [ + "boat.png" + ], + "messages": [ + { + "role": "user", + "question": " Please provide a detailed description of the contents shown in the image.", + "answer": "", + "options": {}, + "choices": [], + "prompt": " Please provide a detailed description of the contents shown in the image.", + "hint": "" + }, + { + "role": "assistant", + "response": "Got it, let's analyze the image. The main subject is a large white yacht or motorboat moving through the water. The boat is sleek, with a modern design, probably a luxury yacht. It's creating a big wake as it moves, which shows it's going at a good speed. The water is" + } + ], + "comment": "Test 'single image start' modality.", + "eval-id": 1 + } +] \ No newline at end of file diff --git a/work_dirs/glm-4d1v/GLM-4.1V-9B-Thinking-text/result.json b/work_dirs/glm-4d1v/GLM-4.1V-9B-Thinking-text/result.json new file mode 100644 index 00000000..040588f2 --- /dev/null +++ b/work_dirs/glm-4d1v/GLM-4.1V-9B-Thinking-text/result.json @@ -0,0 +1,44 @@ +[ + { + "id": 0, + "media": [], + "messages": [ + { + "role": "user", + "question": "What is the result of 1 plus 1?", + "answer": "", + "options": {}, + "choices": [], + "prompt": "What is the result of 1 plus 1?", + "hint": "" + }, + { + "role": "assistant", + "response": "Got it, let's see. The question is asking for the result of 1 plus 1. That's a basic arithmetic operation. When you add" + } + ], + "comment": "Test 'no media' modality.", + "eval-id": 0 + }, + { + "id": 1, + "media": [], + "messages": [ + { + "role": "user", + "question": "What is the capital city of China?", + "answer": "", + "options": {}, + "choices": [], + "prompt": "What is the capital city of China?", + "hint": "" + }, + { + "role": "assistant", + "response": "Got it, the question is asking for the capital city of China. I need to recall that the capital of China is Beijing. Let me make sure there" + } + ], + "comment": "Test 'no media' modality.", + "eval-id": 1 + } +] \ No newline at end of file diff --git a/work_dirs/kimi-vl/Kimi-VL-A3B-Instruct-single-image-start/result.json b/work_dirs/kimi-vl/Kimi-VL-A3B-Instruct-single-image-start/result.json new file mode 100644 index 00000000..801a79dc --- /dev/null +++ b/work_dirs/kimi-vl/Kimi-VL-A3B-Instruct-single-image-start/result.json @@ -0,0 +1,48 @@ +[ + { + "id": 0, + "media": [ + "truck.png" + ], + "messages": [ + { + "role": "user", + "question": " Please provide a detailed description of the contents shown in the image.", + "answer": "", + "options": {}, + "choices": [], + "prompt": " Please provide a detailed description of the contents shown in the image.", + "hint": "" + }, + { + "role": "assistant", + "response": "The image showcases a GMC Sierra 1500 AT4 truck in action, driving through a rugged, desert-like terrain. The truck is painted in a sleek, dark gray color and is equipped with black wheels and off-road tires, emphasizing its robust and adventurous design. The front grille prominently displays the GMC logo, and the" + } + ], + "comment": "Test 'single image start' modality.", + "eval-id": 0 + }, + { + "id": 1, + "media": [ + "boat.png" + ], + "messages": [ + { + "role": "user", + "question": " Please provide a detailed description of the contents shown in the image.", + "answer": "", + "options": {}, + "choices": [], + "prompt": " Please provide a detailed description of the contents shown in the image.", + "hint": "" + }, + { + "role": "assistant", + "response": "The image features a sleek, white luxury yacht cruising through calm blue waters. The yacht is designed with a streamlined shape, enhancing its speed and stability. It has multiple decks, with the upper deck housing what appears to be an open seating area, possibly for relaxation or social gatherings. The lower deck includes a shaded area," + } + ], + "comment": "Test 'single image start' modality.", + "eval-id": 1 + } +] \ No newline at end of file diff --git a/work_dirs/kimi-vl/Kimi-VL-A3B-Instruct-text/result.json b/work_dirs/kimi-vl/Kimi-VL-A3B-Instruct-text/result.json new file mode 100644 index 00000000..19b29031 --- /dev/null +++ b/work_dirs/kimi-vl/Kimi-VL-A3B-Instruct-text/result.json @@ -0,0 +1,44 @@ +[ + { + "id": 0, + "media": [], + "messages": [ + { + "role": "user", + "question": "What is the result of 1 plus 1?", + "answer": "", + "options": {}, + "choices": [], + "prompt": "What is the result of 1 plus 1?", + "hint": "" + }, + { + "role": "assistant", + "response": "The result of 1 plus 1 is 2." + } + ], + "comment": "Test 'no media' modality.", + "eval-id": 0 + }, + { + "id": 1, + "media": [], + "messages": [ + { + "role": "user", + "question": "What is the capital city of China?", + "answer": "", + "options": {}, + "choices": [], + "prompt": "What is the capital city of China?", + "hint": "" + }, + { + "role": "assistant", + "response": "The capital city of China is Beijing." + } + ], + "comment": "Test 'no media' modality.", + "eval-id": 1 + } +] \ No newline at end of file diff --git a/work_dirs/minicpm-v-4d5/MiniCPM-V-4_5-single-image-start/result.json b/work_dirs/minicpm-v-4d5/MiniCPM-V-4_5-single-image-start/result.json new file mode 100644 index 00000000..1dfd25b8 --- /dev/null +++ b/work_dirs/minicpm-v-4d5/MiniCPM-V-4_5-single-image-start/result.json @@ -0,0 +1,48 @@ +[ + { + "id": 0, + "media": [ + "truck.png" + ], + "messages": [ + { + "role": "user", + "question": " Please provide a detailed description of the contents shown in the image.", + "answer": "", + "options": {}, + "choices": [], + "prompt": " Please provide a detailed description of the contents shown in the image.", + "hint": "" + }, + { + "role": "assistant", + "response": "The image features a GMC pickup truck prominently displayed in an outdoor setting, likely on sandy or dusty terrain. The vehicle is captured from a front three-quarter angle, showcasing its bold and robust design. It has a dark metallic gray exterior finish which gives it a sleek yet rugged appearance.\n\nThe front fascia of the truck includes" + } + ], + "comment": "Test 'single image start' modality.", + "eval-id": 0 + }, + { + "id": 1, + "media": [ + "boat.png" + ], + "messages": [ + { + "role": "user", + "question": " Please provide a detailed description of the contents shown in the image.", + "answer": "", + "options": {}, + "choices": [], + "prompt": " Please provide a detailed description of the contents shown in the image.", + "hint": "" + }, + { + "role": "assistant", + "response": "The image depicts a sleek, modern yacht cruising at high speed across the ocean. The vessel is predominantly white with clean lines and an aerodynamic design, emphasizing luxury and performance. It features large windows along the side of the cabin area, suggesting spacious interiors likely equipped with advanced amenities for comfort and entertainment.\n\nThe yacht has two" + } + ], + "comment": "Test 'single image start' modality.", + "eval-id": 1 + } +] \ No newline at end of file diff --git a/work_dirs/minicpm-v-4d5/MiniCPM-V-4_5-text/result.json b/work_dirs/minicpm-v-4d5/MiniCPM-V-4_5-text/result.json new file mode 100644 index 00000000..19b29031 --- /dev/null +++ b/work_dirs/minicpm-v-4d5/MiniCPM-V-4_5-text/result.json @@ -0,0 +1,44 @@ +[ + { + "id": 0, + "media": [], + "messages": [ + { + "role": "user", + "question": "What is the result of 1 plus 1?", + "answer": "", + "options": {}, + "choices": [], + "prompt": "What is the result of 1 plus 1?", + "hint": "" + }, + { + "role": "assistant", + "response": "The result of 1 plus 1 is 2." + } + ], + "comment": "Test 'no media' modality.", + "eval-id": 0 + }, + { + "id": 1, + "media": [], + "messages": [ + { + "role": "user", + "question": "What is the capital city of China?", + "answer": "", + "options": {}, + "choices": [], + "prompt": "What is the capital city of China?", + "hint": "" + }, + { + "role": "assistant", + "response": "The capital city of China is Beijing." + } + ], + "comment": "Test 'no media' modality.", + "eval-id": 1 + } +] \ No newline at end of file diff --git a/work_dirs/ovis2/Ovis2-8B-single-image-start/result.json b/work_dirs/ovis2/Ovis2-8B-single-image-start/result.json new file mode 100644 index 00000000..7e0eda71 --- /dev/null +++ b/work_dirs/ovis2/Ovis2-8B-single-image-start/result.json @@ -0,0 +1,48 @@ +[ + { + "id": 0, + "media": [ + "truck.png" + ], + "messages": [ + { + "role": "user", + "question": " Please provide a detailed description of the contents shown in the image.", + "answer": "", + "options": {}, + "choices": [], + "prompt": " Please provide a detailed description of the contents shown in the image.", + "hint": "" + }, + { + "role": "assistant", + "response": "The image showcases a GMC pickup truck, captured in motion on a dusty, unpaved road, suggesting off-road capabilities. The truck is painted in a sleek, metallic gray, with black accents on the wheels and front grille, enhancing its rugged appearance. The GMC logo is prominently displayed in red on the front grille, adding" + } + ], + "comment": "Test 'single image start' modality.", + "eval-id": 0 + }, + { + "id": 1, + "media": [ + "boat.png" + ], + "messages": [ + { + "role": "user", + "question": " Please provide a detailed description of the contents shown in the image.", + "answer": "", + "options": {}, + "choices": [], + "prompt": " Please provide a detailed description of the contents shown in the image.", + "hint": "" + }, + { + "role": "assistant", + "response": "The image captures a sleek, white yacht gliding through the deep blue ocean. The yacht, with its polished exterior and a black stripe running along its side, is moving swiftly, creating a large wake that churns the water into white foam. The yacht is equipped with a flybridge, visible at the top, and" + } + ], + "comment": "Test 'single image start' modality.", + "eval-id": 1 + } +] \ No newline at end of file diff --git a/work_dirs/ovis2/Ovis2-8B-text/result.json b/work_dirs/ovis2/Ovis2-8B-text/result.json new file mode 100644 index 00000000..19b29031 --- /dev/null +++ b/work_dirs/ovis2/Ovis2-8B-text/result.json @@ -0,0 +1,44 @@ +[ + { + "id": 0, + "media": [], + "messages": [ + { + "role": "user", + "question": "What is the result of 1 plus 1?", + "answer": "", + "options": {}, + "choices": [], + "prompt": "What is the result of 1 plus 1?", + "hint": "" + }, + { + "role": "assistant", + "response": "The result of 1 plus 1 is 2." + } + ], + "comment": "Test 'no media' modality.", + "eval-id": 0 + }, + { + "id": 1, + "media": [], + "messages": [ + { + "role": "user", + "question": "What is the capital city of China?", + "answer": "", + "options": {}, + "choices": [], + "prompt": "What is the capital city of China?", + "hint": "" + }, + { + "role": "assistant", + "response": "The capital city of China is Beijing." + } + ], + "comment": "Test 'no media' modality.", + "eval-id": 1 + } +] \ No newline at end of file From 1d0182cd1a56c95bdf29e92ed43edfc864851c44 Mon Sep 17 00:00:00 2001 From: ZTWHHH Date: Wed, 10 Jun 2026 16:01:47 +0000 Subject: [PATCH 2/6] Add qwen3d5 test scripts and results Test scripts and result.json files for the Qwen3.5 multimodal series, to complement the matching main-branch entry. Layout follows PR #118. Co-Authored-By: Claude Opus 4.7 --- scripts/qwen3d5/qwen3d5-single-image-start.sh | 15 ++++++ scripts/qwen3d5/qwen3d5-text.sh | 14 +++++ .../result.json | 52 +++++++++++++++++++ .../qwen3d5/Qwen3.5-0.8B-text/result.json | 48 +++++++++++++++++ 4 files changed, 129 insertions(+) create mode 100644 scripts/qwen3d5/qwen3d5-single-image-start.sh create mode 100644 scripts/qwen3d5/qwen3d5-text.sh create mode 100644 work_dirs/qwen3d5/Qwen3.5-0.8B-single-image-start/result.json create mode 100644 work_dirs/qwen3d5/Qwen3.5-0.8B-text/result.json diff --git a/scripts/qwen3d5/qwen3d5-single-image-start.sh b/scripts/qwen3d5/qwen3d5-single-image-start.sh new file mode 100644 index 00000000..b72b91c9 --- /dev/null +++ b/scripts/qwen3d5/qwen3d5-single-image-start.sh @@ -0,0 +1,15 @@ +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +MMEVAL_DIR="$SCRIPT_DIR/../../../simple-mmeval-model-dev" +RESULT_DIR="$SCRIPT_DIR/../.." +export PYTHONPATH="$MMEVAL_DIR:$PYTHONPATH" +cd "$MMEVAL_DIR" + +python $MMEVAL_DIR/mmeval/run.py \ + --infile $MMEVAL_DIR/tests/samples/single_image_start.json \ + --dataset local@json \ + --out_dir $RESULT_DIR/work_dirs/qwen3d5/Qwen3.5-0.8B-single-image-start \ + --img_dir $MMEVAL_DIR/tests/media/448 \ + --model_name_or_path Qwen/Qwen3.5-0.8B \ + --gpu_per_parallel 1 \ + --parallel_per_task 1 \ + --max_new_tokens 512 diff --git a/scripts/qwen3d5/qwen3d5-text.sh b/scripts/qwen3d5/qwen3d5-text.sh new file mode 100644 index 00000000..dc43e4e6 --- /dev/null +++ b/scripts/qwen3d5/qwen3d5-text.sh @@ -0,0 +1,14 @@ +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +MMEVAL_DIR="$SCRIPT_DIR/../../../simple-mmeval-model-dev" +RESULT_DIR="$SCRIPT_DIR/../.." +export PYTHONPATH="$MMEVAL_DIR:$PYTHONPATH" +cd "$MMEVAL_DIR" + +python $MMEVAL_DIR/mmeval/run.py \ + --infile $MMEVAL_DIR/tests/samples/no_media.json \ + --dataset local@json \ + --out_dir $RESULT_DIR/work_dirs/qwen3d5/Qwen3.5-0.8B-text \ + --model_name_or_path Qwen/Qwen3.5-0.8B \ + --gpu_per_parallel 1 \ + --parallel_per_task 1 \ + --max_new_tokens 512 diff --git a/work_dirs/qwen3d5/Qwen3.5-0.8B-single-image-start/result.json b/work_dirs/qwen3d5/Qwen3.5-0.8B-single-image-start/result.json new file mode 100644 index 00000000..b9fa4c7b --- /dev/null +++ b/work_dirs/qwen3d5/Qwen3.5-0.8B-single-image-start/result.json @@ -0,0 +1,52 @@ +[ + { + "id": 0, + "media": [ + "truck.png" + ], + "messages": [ + { + "role": "user", + "question": " Please provide a detailed description of the contents shown in the image.", + "answer": "", + "options": {}, + "choices": [], + "prompt": " Please provide a detailed description of the contents shown in the image.", + "hint": "" + }, + { + "role": "assistant", + "response": [ + "This is a dynamic, action-oriented photograph of a **2024 GMC Sierra 1500 pickup truck** in motion on a dusty, arid terrain. Here\u2019s a detailed breakdown:\n\n---\n\n### **Vehicle Details:**\n\n- **Make & Model**: GMC Sierra 1500 (" + ] + } + ], + "comment": "Test 'single image start' modality.", + "eval-id": 0 + }, + { + "id": 1, + "media": [ + "boat.png" + ], + "messages": [ + { + "role": "user", + "question": " Please provide a detailed description of the contents shown in the image.", + "answer": "", + "options": {}, + "choices": [], + "prompt": " Please provide a detailed description of the contents shown in the image.", + "hint": "" + }, + { + "role": "assistant", + "response": [ + "This is a dynamic, wide-angle photograph capturing a white speedboat cutting through the ocean. Here\u2019s a detailed description:\n\n**Subject:**\n- The boat is a modern, high-performance motor yacht, predominantly white with a sleek, aerodynamic design.\n- It features a large, curved windshield and a raised helm" + ] + } + ], + "comment": "Test 'single image start' modality.", + "eval-id": 1 + } +] \ No newline at end of file diff --git a/work_dirs/qwen3d5/Qwen3.5-0.8B-text/result.json b/work_dirs/qwen3d5/Qwen3.5-0.8B-text/result.json new file mode 100644 index 00000000..be621c11 --- /dev/null +++ b/work_dirs/qwen3d5/Qwen3.5-0.8B-text/result.json @@ -0,0 +1,48 @@ +[ + { + "id": 0, + "media": [], + "messages": [ + { + "role": "user", + "question": "What is the result of 1 plus 1?", + "answer": "", + "options": {}, + "choices": [], + "prompt": "What is the result of 1 plus 1?", + "hint": "" + }, + { + "role": "assistant", + "response": [ + "The result of **1 plus 1** is **2**.\n\nMathematically, this is expressed as:\n$$1 + 1 = 2$$\n\nIn the context of the number line, 1 is one unit to the right of zero, and adding 1 brings it one unit further to the" + ] + } + ], + "comment": "Test 'no media' modality.", + "eval-id": 0 + }, + { + "id": 1, + "media": [], + "messages": [ + { + "role": "user", + "question": "What is the capital city of China?", + "answer": "", + "options": {}, + "choices": [], + "prompt": "What is the capital city of China?", + "hint": "" + }, + { + "role": "assistant", + "response": [ + "The capital city of China is **Beijing**. It is the largest city in China and serves as the country's political, economic, cultural, and scientific center.\n" + ] + } + ], + "comment": "Test 'no media' modality.", + "eval-id": 1 + } +] \ No newline at end of file From 38f46b4b1b771abb2f627ca60803d1caa145aae3 Mon Sep 17 00:00:00 2001 From: ZTWHHH Date: Thu, 11 Jun 2026 07:52:14 +0000 Subject: [PATCH 3/6] Switch test modalities to broadest correct per series; test every variant Per official capability: - kimi_vl: text + multi-image-start. Both Kimi-VL-A3B-{Instruct,Thinking}. - minicpm_v_4d5: text + multi-image-start. MiniCPM-V-4_5. - glm_4d1v: text + single-video-start. GLM-4.1V-9B-{Thinking,Base}. - ovis2: text + multi-image-start. Ovis2-{1B,2B,4B,8B}. - aria: text + multi-image-start. Aria. - qwen3d5: text + multi-image-video-interleave. Qwen3.5-{0.8B,2B,4B,9B}. Each variant has both a text result.json and a broader-modality result.json under work_dirs//-/. Single-image tests were dropped in favor of multi-image-start (which fully covers single-image). --- ...age-start.sh => aria-multi-image-start.sh} | 5 +- scripts/aria/aria-text.sh | 1 + ...tart.sh => glm-4d1v-single-video-start.sh} | 15 +- scripts/glm-4d1v/glm-4d1v-text.sh | 10 + ...-start.sh => kimi-vl-multi-image-start.sh} | 15 +- scripts/kimi-vl/kimi-vl-text.sh | 10 + ....sh => minicpm-v-4d5-multi-image-start.sh} | 5 +- scripts/minicpm-v-4d5/minicpm-v-4d5-text.sh | 1 + scripts/ovis2/ovis2-multi-image-start.sh | 46 ++ scripts/ovis2/ovis2-single-image-start.sh | 15 - scripts/ovis2/ovis2-text.sh | 28 ++ .../qwen3d5-multi-image-video-interleave.sh | 46 ++ scripts/qwen3d5/qwen3d5-single-image-start.sh | 15 - scripts/qwen3d5/qwen3d5-text.sh | 28 ++ .../aria/Aria-multi-image-start/result.json | 96 ++++ .../aria/Aria-single-image-start/result.json | 48 -- work_dirs/aria/Aria-text/result.json | 2 +- .../result.json | 94 ++++ .../GLM-4.1V-9B-Base-text/result.json | 44 ++ .../result.json | 48 -- .../result.json | 94 ++++ .../GLM-4.1V-9B-Thinking-text/result.json | 2 +- .../result.json | 96 ++++ .../result.json | 48 -- .../Kimi-VL-A3B-Instruct-text/result.json | 2 +- .../result.json | 96 ++++ .../Kimi-VL-A3B-Thinking-text/result.json | 44 ++ .../result.json | 96 ++++ .../result.json | 48 -- .../MiniCPM-V-4_5-text/result.json | 2 +- .../Ovis2-1B-multi-image-start/result.json | 96 ++++ work_dirs/ovis2/Ovis2-1B-text/result.json | 44 ++ .../Ovis2-2B-multi-image-start/result.json | 96 ++++ work_dirs/ovis2/Ovis2-2B-text/result.json | 44 ++ .../Ovis2-4B-multi-image-start/result.json | 96 ++++ work_dirs/ovis2/Ovis2-4B-text/result.json | 44 ++ .../Ovis2-8B-multi-image-start/result.json | 96 ++++ .../Ovis2-8B-single-image-start/result.json | 48 -- .../result.json | 466 ++++++++++++++++++ .../result.json | 52 -- .../qwen3d5/Qwen3.5-0.8B-text/result.json | 4 +- .../result.json | 466 ++++++++++++++++++ work_dirs/qwen3d5/Qwen3.5-2B-text/result.json | 48 ++ .../result.json | 466 ++++++++++++++++++ work_dirs/qwen3d5/Qwen3.5-4B-text/result.json | 48 ++ .../result.json | 466 ++++++++++++++++++ work_dirs/qwen3d5/Qwen3.5-9B-text/result.json | 48 ++ 47 files changed, 3392 insertions(+), 336 deletions(-) rename scripts/aria/{aria-single-image-start.sh => aria-multi-image-start.sh} (76%) rename scripts/glm-4d1v/{glm-4d1v-single-image-start.sh => glm-4d1v-single-video-start.sh} (51%) rename scripts/kimi-vl/{kimi-vl-single-image-start.sh => kimi-vl-multi-image-start.sh} (51%) rename scripts/minicpm-v-4d5/{minicpm-v-4d5-single-image-start.sh => minicpm-v-4d5-multi-image-start.sh} (85%) create mode 100644 scripts/ovis2/ovis2-multi-image-start.sh delete mode 100644 scripts/ovis2/ovis2-single-image-start.sh create mode 100644 scripts/qwen3d5/qwen3d5-multi-image-video-interleave.sh delete mode 100644 scripts/qwen3d5/qwen3d5-single-image-start.sh create mode 100644 work_dirs/aria/Aria-multi-image-start/result.json delete mode 100644 work_dirs/aria/Aria-single-image-start/result.json create mode 100644 work_dirs/glm-4d1v/GLM-4.1V-9B-Base-single-video-start/result.json create mode 100644 work_dirs/glm-4d1v/GLM-4.1V-9B-Base-text/result.json delete mode 100644 work_dirs/glm-4d1v/GLM-4.1V-9B-Thinking-single-image-start/result.json create mode 100644 work_dirs/glm-4d1v/GLM-4.1V-9B-Thinking-single-video-start/result.json create mode 100644 work_dirs/kimi-vl/Kimi-VL-A3B-Instruct-multi-image-start/result.json delete mode 100644 work_dirs/kimi-vl/Kimi-VL-A3B-Instruct-single-image-start/result.json create mode 100644 work_dirs/kimi-vl/Kimi-VL-A3B-Thinking-multi-image-start/result.json create mode 100644 work_dirs/kimi-vl/Kimi-VL-A3B-Thinking-text/result.json create mode 100644 work_dirs/minicpm-v-4d5/MiniCPM-V-4_5-multi-image-start/result.json delete mode 100644 work_dirs/minicpm-v-4d5/MiniCPM-V-4_5-single-image-start/result.json create mode 100644 work_dirs/ovis2/Ovis2-1B-multi-image-start/result.json create mode 100644 work_dirs/ovis2/Ovis2-1B-text/result.json create mode 100644 work_dirs/ovis2/Ovis2-2B-multi-image-start/result.json create mode 100644 work_dirs/ovis2/Ovis2-2B-text/result.json create mode 100644 work_dirs/ovis2/Ovis2-4B-multi-image-start/result.json create mode 100644 work_dirs/ovis2/Ovis2-4B-text/result.json create mode 100644 work_dirs/ovis2/Ovis2-8B-multi-image-start/result.json delete mode 100644 work_dirs/ovis2/Ovis2-8B-single-image-start/result.json create mode 100644 work_dirs/qwen3d5/Qwen3.5-0.8B-multi-image-video-interleave/result.json delete mode 100644 work_dirs/qwen3d5/Qwen3.5-0.8B-single-image-start/result.json create mode 100644 work_dirs/qwen3d5/Qwen3.5-2B-multi-image-video-interleave/result.json create mode 100644 work_dirs/qwen3d5/Qwen3.5-2B-text/result.json create mode 100644 work_dirs/qwen3d5/Qwen3.5-4B-multi-image-video-interleave/result.json create mode 100644 work_dirs/qwen3d5/Qwen3.5-4B-text/result.json create mode 100644 work_dirs/qwen3d5/Qwen3.5-9B-multi-image-video-interleave/result.json create mode 100644 work_dirs/qwen3d5/Qwen3.5-9B-text/result.json diff --git a/scripts/aria/aria-single-image-start.sh b/scripts/aria/aria-multi-image-start.sh similarity index 76% rename from scripts/aria/aria-single-image-start.sh rename to scripts/aria/aria-multi-image-start.sh index ff5b83a0..1abb7333 100644 --- a/scripts/aria/aria-single-image-start.sh +++ b/scripts/aria/aria-multi-image-start.sh @@ -5,11 +5,12 @@ export PYTHONPATH="$MMEVAL_DIR:$PYTHONPATH" cd "$MMEVAL_DIR" python $MMEVAL_DIR/mmeval/run.py \ - --infile $MMEVAL_DIR/tests/samples/single_image_start.json \ + --infile $MMEVAL_DIR/tests/samples/multi_image_start.json \ --dataset local@json \ - --out_dir $RESULT_DIR/work_dirs/aria/Aria-single-image-start \ + --out_dir $RESULT_DIR/work_dirs/aria/Aria-multi-image-start \ --img_dir $MMEVAL_DIR/tests/media/448 \ --model_name_or_path rhymes-ai/Aria \ --gpu_per_parallel 1 \ --parallel_per_task 1 \ --max_new_tokens 512 + diff --git a/scripts/aria/aria-text.sh b/scripts/aria/aria-text.sh index 1c96ce52..98fed105 100644 --- a/scripts/aria/aria-text.sh +++ b/scripts/aria/aria-text.sh @@ -12,3 +12,4 @@ python $MMEVAL_DIR/mmeval/run.py \ --gpu_per_parallel 1 \ --parallel_per_task 1 \ --max_new_tokens 512 + diff --git a/scripts/glm-4d1v/glm-4d1v-single-image-start.sh b/scripts/glm-4d1v/glm-4d1v-single-video-start.sh similarity index 51% rename from scripts/glm-4d1v/glm-4d1v-single-image-start.sh rename to scripts/glm-4d1v/glm-4d1v-single-video-start.sh index baeee74a..6a777829 100644 --- a/scripts/glm-4d1v/glm-4d1v-single-image-start.sh +++ b/scripts/glm-4d1v/glm-4d1v-single-video-start.sh @@ -5,11 +5,22 @@ export PYTHONPATH="$MMEVAL_DIR:$PYTHONPATH" cd "$MMEVAL_DIR" python $MMEVAL_DIR/mmeval/run.py \ - --infile $MMEVAL_DIR/tests/samples/single_image_start.json \ + --infile $MMEVAL_DIR/tests/samples/single_video_start.json \ --dataset local@json \ - --out_dir $RESULT_DIR/work_dirs/glm-4d1v/GLM-4.1V-9B-Thinking-single-image-start \ + --out_dir $RESULT_DIR/work_dirs/glm-4d1v/GLM-4.1V-9B-Thinking-single-video-start \ --img_dir $MMEVAL_DIR/tests/media/448 \ --model_name_or_path THUDM/GLM-4.1V-9B-Thinking \ --gpu_per_parallel 1 \ --parallel_per_task 1 \ --max_new_tokens 512 + +python $MMEVAL_DIR/mmeval/run.py \ + --infile $MMEVAL_DIR/tests/samples/single_video_start.json \ + --dataset local@json \ + --out_dir $RESULT_DIR/work_dirs/glm-4d1v/GLM-4.1V-9B-Base-single-video-start \ + --img_dir $MMEVAL_DIR/tests/media/448 \ + --model_name_or_path THUDM/GLM-4.1V-9B-Base \ + --gpu_per_parallel 1 \ + --parallel_per_task 1 \ + --max_new_tokens 512 + diff --git a/scripts/glm-4d1v/glm-4d1v-text.sh b/scripts/glm-4d1v/glm-4d1v-text.sh index a2f58e02..7206a2ed 100644 --- a/scripts/glm-4d1v/glm-4d1v-text.sh +++ b/scripts/glm-4d1v/glm-4d1v-text.sh @@ -12,3 +12,13 @@ python $MMEVAL_DIR/mmeval/run.py \ --gpu_per_parallel 1 \ --parallel_per_task 1 \ --max_new_tokens 512 + +python $MMEVAL_DIR/mmeval/run.py \ + --infile $MMEVAL_DIR/tests/samples/no_media.json \ + --dataset local@json \ + --out_dir $RESULT_DIR/work_dirs/glm-4d1v/GLM-4.1V-9B-Base-text \ + --model_name_or_path THUDM/GLM-4.1V-9B-Base \ + --gpu_per_parallel 1 \ + --parallel_per_task 1 \ + --max_new_tokens 512 + diff --git a/scripts/kimi-vl/kimi-vl-single-image-start.sh b/scripts/kimi-vl/kimi-vl-multi-image-start.sh similarity index 51% rename from scripts/kimi-vl/kimi-vl-single-image-start.sh rename to scripts/kimi-vl/kimi-vl-multi-image-start.sh index 9caafbb7..04b55da3 100644 --- a/scripts/kimi-vl/kimi-vl-single-image-start.sh +++ b/scripts/kimi-vl/kimi-vl-multi-image-start.sh @@ -5,11 +5,22 @@ export PYTHONPATH="$MMEVAL_DIR:$PYTHONPATH" cd "$MMEVAL_DIR" python $MMEVAL_DIR/mmeval/run.py \ - --infile $MMEVAL_DIR/tests/samples/single_image_start.json \ + --infile $MMEVAL_DIR/tests/samples/multi_image_start.json \ --dataset local@json \ - --out_dir $RESULT_DIR/work_dirs/kimi-vl/Kimi-VL-A3B-Instruct-single-image-start \ + --out_dir $RESULT_DIR/work_dirs/kimi-vl/Kimi-VL-A3B-Instruct-multi-image-start \ --img_dir $MMEVAL_DIR/tests/media/448 \ --model_name_or_path moonshotai/Kimi-VL-A3B-Instruct \ --gpu_per_parallel 1 \ --parallel_per_task 1 \ --max_new_tokens 512 + +python $MMEVAL_DIR/mmeval/run.py \ + --infile $MMEVAL_DIR/tests/samples/multi_image_start.json \ + --dataset local@json \ + --out_dir $RESULT_DIR/work_dirs/kimi-vl/Kimi-VL-A3B-Thinking-multi-image-start \ + --img_dir $MMEVAL_DIR/tests/media/448 \ + --model_name_or_path moonshotai/Kimi-VL-A3B-Thinking \ + --gpu_per_parallel 1 \ + --parallel_per_task 1 \ + --max_new_tokens 512 + diff --git a/scripts/kimi-vl/kimi-vl-text.sh b/scripts/kimi-vl/kimi-vl-text.sh index 67d78d23..81377241 100644 --- a/scripts/kimi-vl/kimi-vl-text.sh +++ b/scripts/kimi-vl/kimi-vl-text.sh @@ -12,3 +12,13 @@ python $MMEVAL_DIR/mmeval/run.py \ --gpu_per_parallel 1 \ --parallel_per_task 1 \ --max_new_tokens 512 + +python $MMEVAL_DIR/mmeval/run.py \ + --infile $MMEVAL_DIR/tests/samples/no_media.json \ + --dataset local@json \ + --out_dir $RESULT_DIR/work_dirs/kimi-vl/Kimi-VL-A3B-Thinking-text \ + --model_name_or_path moonshotai/Kimi-VL-A3B-Thinking \ + --gpu_per_parallel 1 \ + --parallel_per_task 1 \ + --max_new_tokens 512 + diff --git a/scripts/minicpm-v-4d5/minicpm-v-4d5-single-image-start.sh b/scripts/minicpm-v-4d5/minicpm-v-4d5-multi-image-start.sh similarity index 85% rename from scripts/minicpm-v-4d5/minicpm-v-4d5-single-image-start.sh rename to scripts/minicpm-v-4d5/minicpm-v-4d5-multi-image-start.sh index 71078ca6..b8a9be82 100644 --- a/scripts/minicpm-v-4d5/minicpm-v-4d5-single-image-start.sh +++ b/scripts/minicpm-v-4d5/minicpm-v-4d5-multi-image-start.sh @@ -5,11 +5,12 @@ export PYTHONPATH="$MMEVAL_DIR:$PYTHONPATH" cd "$MMEVAL_DIR" python $MMEVAL_DIR/mmeval/run.py \ - --infile $MMEVAL_DIR/tests/samples/single_image_start.json \ + --infile $MMEVAL_DIR/tests/samples/multi_image_start.json \ --dataset local@json \ - --out_dir $RESULT_DIR/work_dirs/minicpm-v-4d5/MiniCPM-V-4_5-single-image-start \ + --out_dir $RESULT_DIR/work_dirs/minicpm-v-4d5/MiniCPM-V-4_5-multi-image-start \ --img_dir $MMEVAL_DIR/tests/media/448 \ --model_name_or_path openbmb/MiniCPM-V-4_5 \ --gpu_per_parallel 1 \ --parallel_per_task 1 \ --max_new_tokens 512 + diff --git a/scripts/minicpm-v-4d5/minicpm-v-4d5-text.sh b/scripts/minicpm-v-4d5/minicpm-v-4d5-text.sh index ba465450..4a476461 100644 --- a/scripts/minicpm-v-4d5/minicpm-v-4d5-text.sh +++ b/scripts/minicpm-v-4d5/minicpm-v-4d5-text.sh @@ -12,3 +12,4 @@ python $MMEVAL_DIR/mmeval/run.py \ --gpu_per_parallel 1 \ --parallel_per_task 1 \ --max_new_tokens 512 + diff --git a/scripts/ovis2/ovis2-multi-image-start.sh b/scripts/ovis2/ovis2-multi-image-start.sh new file mode 100644 index 00000000..2bfe8a0e --- /dev/null +++ b/scripts/ovis2/ovis2-multi-image-start.sh @@ -0,0 +1,46 @@ +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +MMEVAL_DIR="$SCRIPT_DIR/../../../simple-mmeval-model-dev" +RESULT_DIR="$SCRIPT_DIR/../.." +export PYTHONPATH="$MMEVAL_DIR:$PYTHONPATH" +cd "$MMEVAL_DIR" + +python $MMEVAL_DIR/mmeval/run.py \ + --infile $MMEVAL_DIR/tests/samples/multi_image_start.json \ + --dataset local@json \ + --out_dir $RESULT_DIR/work_dirs/ovis2/Ovis2-1B-multi-image-start \ + --img_dir $MMEVAL_DIR/tests/media/448 \ + --model_name_or_path AIDC-AI/Ovis2-1B \ + --gpu_per_parallel 1 \ + --parallel_per_task 1 \ + --max_new_tokens 512 + +python $MMEVAL_DIR/mmeval/run.py \ + --infile $MMEVAL_DIR/tests/samples/multi_image_start.json \ + --dataset local@json \ + --out_dir $RESULT_DIR/work_dirs/ovis2/Ovis2-2B-multi-image-start \ + --img_dir $MMEVAL_DIR/tests/media/448 \ + --model_name_or_path AIDC-AI/Ovis2-2B \ + --gpu_per_parallel 1 \ + --parallel_per_task 1 \ + --max_new_tokens 512 + +python $MMEVAL_DIR/mmeval/run.py \ + --infile $MMEVAL_DIR/tests/samples/multi_image_start.json \ + --dataset local@json \ + --out_dir $RESULT_DIR/work_dirs/ovis2/Ovis2-4B-multi-image-start \ + --img_dir $MMEVAL_DIR/tests/media/448 \ + --model_name_or_path AIDC-AI/Ovis2-4B \ + --gpu_per_parallel 1 \ + --parallel_per_task 1 \ + --max_new_tokens 512 + +python $MMEVAL_DIR/mmeval/run.py \ + --infile $MMEVAL_DIR/tests/samples/multi_image_start.json \ + --dataset local@json \ + --out_dir $RESULT_DIR/work_dirs/ovis2/Ovis2-8B-multi-image-start \ + --img_dir $MMEVAL_DIR/tests/media/448 \ + --model_name_or_path AIDC-AI/Ovis2-8B \ + --gpu_per_parallel 1 \ + --parallel_per_task 1 \ + --max_new_tokens 512 + diff --git a/scripts/ovis2/ovis2-single-image-start.sh b/scripts/ovis2/ovis2-single-image-start.sh deleted file mode 100644 index d720bb39..00000000 --- a/scripts/ovis2/ovis2-single-image-start.sh +++ /dev/null @@ -1,15 +0,0 @@ -SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" -MMEVAL_DIR="$SCRIPT_DIR/../../../simple-mmeval-model-dev" -RESULT_DIR="$SCRIPT_DIR/../.." -export PYTHONPATH="$MMEVAL_DIR:$PYTHONPATH" -cd "$MMEVAL_DIR" - -python $MMEVAL_DIR/mmeval/run.py \ - --infile $MMEVAL_DIR/tests/samples/single_image_start.json \ - --dataset local@json \ - --out_dir $RESULT_DIR/work_dirs/ovis2/Ovis2-8B-single-image-start \ - --img_dir $MMEVAL_DIR/tests/media/448 \ - --model_name_or_path AIDC-AI/Ovis2-8B \ - --gpu_per_parallel 1 \ - --parallel_per_task 1 \ - --max_new_tokens 512 diff --git a/scripts/ovis2/ovis2-text.sh b/scripts/ovis2/ovis2-text.sh index cb2190a6..ca288e60 100644 --- a/scripts/ovis2/ovis2-text.sh +++ b/scripts/ovis2/ovis2-text.sh @@ -4,6 +4,33 @@ RESULT_DIR="$SCRIPT_DIR/../.." export PYTHONPATH="$MMEVAL_DIR:$PYTHONPATH" cd "$MMEVAL_DIR" +python $MMEVAL_DIR/mmeval/run.py \ + --infile $MMEVAL_DIR/tests/samples/no_media.json \ + --dataset local@json \ + --out_dir $RESULT_DIR/work_dirs/ovis2/Ovis2-1B-text \ + --model_name_or_path AIDC-AI/Ovis2-1B \ + --gpu_per_parallel 1 \ + --parallel_per_task 1 \ + --max_new_tokens 512 + +python $MMEVAL_DIR/mmeval/run.py \ + --infile $MMEVAL_DIR/tests/samples/no_media.json \ + --dataset local@json \ + --out_dir $RESULT_DIR/work_dirs/ovis2/Ovis2-2B-text \ + --model_name_or_path AIDC-AI/Ovis2-2B \ + --gpu_per_parallel 1 \ + --parallel_per_task 1 \ + --max_new_tokens 512 + +python $MMEVAL_DIR/mmeval/run.py \ + --infile $MMEVAL_DIR/tests/samples/no_media.json \ + --dataset local@json \ + --out_dir $RESULT_DIR/work_dirs/ovis2/Ovis2-4B-text \ + --model_name_or_path AIDC-AI/Ovis2-4B \ + --gpu_per_parallel 1 \ + --parallel_per_task 1 \ + --max_new_tokens 512 + python $MMEVAL_DIR/mmeval/run.py \ --infile $MMEVAL_DIR/tests/samples/no_media.json \ --dataset local@json \ @@ -12,3 +39,4 @@ python $MMEVAL_DIR/mmeval/run.py \ --gpu_per_parallel 1 \ --parallel_per_task 1 \ --max_new_tokens 512 + diff --git a/scripts/qwen3d5/qwen3d5-multi-image-video-interleave.sh b/scripts/qwen3d5/qwen3d5-multi-image-video-interleave.sh new file mode 100644 index 00000000..c303a9e9 --- /dev/null +++ b/scripts/qwen3d5/qwen3d5-multi-image-video-interleave.sh @@ -0,0 +1,46 @@ +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +MMEVAL_DIR="$SCRIPT_DIR/../../../simple-mmeval-model-dev" +RESULT_DIR="$SCRIPT_DIR/../.." +export PYTHONPATH="$MMEVAL_DIR:$PYTHONPATH" +cd "$MMEVAL_DIR" + +python $MMEVAL_DIR/mmeval/run.py \ + --infile $MMEVAL_DIR/tests/samples/multi_image_video_interleave.json \ + --dataset local@json \ + --out_dir $RESULT_DIR/work_dirs/qwen3d5/Qwen3.5-0.8B-multi-image-video-interleave \ + --img_dir $MMEVAL_DIR/tests/media/448 \ + --model_name_or_path Qwen/Qwen3.5-0.8B \ + --gpu_per_parallel 1 \ + --parallel_per_task 1 \ + --max_new_tokens 512 + +python $MMEVAL_DIR/mmeval/run.py \ + --infile $MMEVAL_DIR/tests/samples/multi_image_video_interleave.json \ + --dataset local@json \ + --out_dir $RESULT_DIR/work_dirs/qwen3d5/Qwen3.5-2B-multi-image-video-interleave \ + --img_dir $MMEVAL_DIR/tests/media/448 \ + --model_name_or_path Qwen/Qwen3.5-2B \ + --gpu_per_parallel 1 \ + --parallel_per_task 1 \ + --max_new_tokens 512 + +python $MMEVAL_DIR/mmeval/run.py \ + --infile $MMEVAL_DIR/tests/samples/multi_image_video_interleave.json \ + --dataset local@json \ + --out_dir $RESULT_DIR/work_dirs/qwen3d5/Qwen3.5-4B-multi-image-video-interleave \ + --img_dir $MMEVAL_DIR/tests/media/448 \ + --model_name_or_path Qwen/Qwen3.5-4B \ + --gpu_per_parallel 1 \ + --parallel_per_task 1 \ + --max_new_tokens 512 + +python $MMEVAL_DIR/mmeval/run.py \ + --infile $MMEVAL_DIR/tests/samples/multi_image_video_interleave.json \ + --dataset local@json \ + --out_dir $RESULT_DIR/work_dirs/qwen3d5/Qwen3.5-9B-multi-image-video-interleave \ + --img_dir $MMEVAL_DIR/tests/media/448 \ + --model_name_or_path Qwen/Qwen3.5-9B \ + --gpu_per_parallel 1 \ + --parallel_per_task 1 \ + --max_new_tokens 512 + diff --git a/scripts/qwen3d5/qwen3d5-single-image-start.sh b/scripts/qwen3d5/qwen3d5-single-image-start.sh deleted file mode 100644 index b72b91c9..00000000 --- a/scripts/qwen3d5/qwen3d5-single-image-start.sh +++ /dev/null @@ -1,15 +0,0 @@ -SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" -MMEVAL_DIR="$SCRIPT_DIR/../../../simple-mmeval-model-dev" -RESULT_DIR="$SCRIPT_DIR/../.." -export PYTHONPATH="$MMEVAL_DIR:$PYTHONPATH" -cd "$MMEVAL_DIR" - -python $MMEVAL_DIR/mmeval/run.py \ - --infile $MMEVAL_DIR/tests/samples/single_image_start.json \ - --dataset local@json \ - --out_dir $RESULT_DIR/work_dirs/qwen3d5/Qwen3.5-0.8B-single-image-start \ - --img_dir $MMEVAL_DIR/tests/media/448 \ - --model_name_or_path Qwen/Qwen3.5-0.8B \ - --gpu_per_parallel 1 \ - --parallel_per_task 1 \ - --max_new_tokens 512 diff --git a/scripts/qwen3d5/qwen3d5-text.sh b/scripts/qwen3d5/qwen3d5-text.sh index dc43e4e6..6dc7dc04 100644 --- a/scripts/qwen3d5/qwen3d5-text.sh +++ b/scripts/qwen3d5/qwen3d5-text.sh @@ -12,3 +12,31 @@ python $MMEVAL_DIR/mmeval/run.py \ --gpu_per_parallel 1 \ --parallel_per_task 1 \ --max_new_tokens 512 + +python $MMEVAL_DIR/mmeval/run.py \ + --infile $MMEVAL_DIR/tests/samples/no_media.json \ + --dataset local@json \ + --out_dir $RESULT_DIR/work_dirs/qwen3d5/Qwen3.5-2B-text \ + --model_name_or_path Qwen/Qwen3.5-2B \ + --gpu_per_parallel 1 \ + --parallel_per_task 1 \ + --max_new_tokens 512 + +python $MMEVAL_DIR/mmeval/run.py \ + --infile $MMEVAL_DIR/tests/samples/no_media.json \ + --dataset local@json \ + --out_dir $RESULT_DIR/work_dirs/qwen3d5/Qwen3.5-4B-text \ + --model_name_or_path Qwen/Qwen3.5-4B \ + --gpu_per_parallel 1 \ + --parallel_per_task 1 \ + --max_new_tokens 512 + +python $MMEVAL_DIR/mmeval/run.py \ + --infile $MMEVAL_DIR/tests/samples/no_media.json \ + --dataset local@json \ + --out_dir $RESULT_DIR/work_dirs/qwen3d5/Qwen3.5-9B-text \ + --model_name_or_path Qwen/Qwen3.5-9B \ + --gpu_per_parallel 1 \ + --parallel_per_task 1 \ + --max_new_tokens 512 + diff --git a/work_dirs/aria/Aria-multi-image-start/result.json b/work_dirs/aria/Aria-multi-image-start/result.json new file mode 100644 index 00000000..7607f9d8 --- /dev/null +++ b/work_dirs/aria/Aria-multi-image-start/result.json @@ -0,0 +1,96 @@ +[ + { + "id": 0, + "media": [ + "truck.png" + ], + "messages": [ + { + "role": "user", + "question": " Please provide a detailed description of the contents shown in the image.", + "answer": "", + "options": {}, + "choices": [], + "prompt": " Please provide a detailed description of the contents shown in the image.", + "hint": "" + }, + { + "role": "assistant", + "response": "The image features a GMC truck driving on a rugged, off-road terrain. The truck appears to be a four-door crew cab design with a robust build, suitable for heavy-duty use. The vehicle is painted in a sleek, dark color that shines under the light, complemented by" + } + ], + "comment": "Test 'single image start' modality, 'multi image start' modality should be able to handle this.", + "eval-id": 0 + }, + { + "id": 1, + "media": [ + "boat.png" + ], + "messages": [ + { + "role": "user", + "question": " Please provide a detailed description of the contents shown in the image.", + "answer": "", + "options": {}, + "choices": [], + "prompt": " Please provide a detailed description of the contents shown in the image.", + "hint": "" + }, + { + "role": "assistant", + "response": " The image shows a sleek, modern yacht speeding through the ocean. The yacht is predominantly white with black trim around the windows and along the hull. It has multiple decks, including a lower deck and an upper deck with a cabin and windows. The yacht appears to be in motion" + } + ], + "comment": "Test 'single image start' modality, 'multi image start' modality should be able to handle this.", + "eval-id": 1 + }, + { + "id": 2, + "media": [ + "truck.png", + "boat.png" + ], + "messages": [ + { + "role": "user", + "question": " Please provide a detailed description of the contents shown in the two images.", + "answer": "", + "options": {}, + "choices": [], + "prompt": " Please provide a detailed description of the contents shown in the two images.", + "hint": "" + }, + { + "role": "assistant", + "response": "The first image features a GMC truck, likely an off-road model, driving through a dusty and rugged outdoors environment. The truck is depicted at an angle, showcasing its front grille and headlights, with the GMC logo prominently displayed. The wheels are large and rugged, suited" + } + ], + "comment": "Test 'multi image start' modality.", + "eval-id": 2 + }, + { + "id": 3, + "media": [ + "boat.png", + "truck.png" + ], + "messages": [ + { + "role": "user", + "question": " Please provide a detailed description of the contents shown in the two images.", + "answer": "", + "options": {}, + "choices": [], + "prompt": " Please provide a detailed description of the contents shown in the two images.", + "hint": "" + }, + { + "role": "assistant", + "response": "The first image depicts a sleek white yacht cutting through the water, creating a beautiful wake behind it. The sky is clear, and the sea appears calm, making it a serene scene of luxury cruising. The yacht seems modern, with a streamlined design and a spacious deck, suggesting it is" + } + ], + "comment": "Test 'multi image start' modality.", + "eval-id": 3 + } +] \ No newline at end of file diff --git a/work_dirs/aria/Aria-single-image-start/result.json b/work_dirs/aria/Aria-single-image-start/result.json deleted file mode 100644 index 55545349..00000000 --- a/work_dirs/aria/Aria-single-image-start/result.json +++ /dev/null @@ -1,48 +0,0 @@ -[ - { - "id": 0, - "media": [ - "truck.png" - ], - "messages": [ - { - "role": "user", - "question": " Please provide a detailed description of the contents shown in the image.", - "answer": "", - "options": {}, - "choices": [], - "prompt": " Please provide a detailed description of the contents shown in the image.", - "hint": "" - }, - { - "role": "assistant", - "response": "The image shows a powerful GMC pickup truck in motion, driving off-road. It's a four-door model with a sleek, modern design. The truck is a metallic shade of gray, with black rims and side mirrors. The terrain beneath it is rough, dry dirt, and it is kicking" - } - ], - "comment": "Test 'single image start' modality.", - "eval-id": 0 - }, - { - "id": 1, - "media": [ - "boat.png" - ], - "messages": [ - { - "role": "user", - "question": " Please provide a detailed description of the contents shown in the image.", - "answer": "", - "options": {}, - "choices": [], - "prompt": " Please provide a detailed description of the contents shown in the image.", - "hint": "" - }, - { - "role": "assistant", - "response": "The image shows a sleek, white motor yacht cruising through calm seas under a clear blue sky. The yacht features a modern design with a low profile and a streamlined hull. There is a prominent bow wave created as the boat cuts through the water, indicating it is moving at a high speed." - } - ], - "comment": "Test 'single image start' modality.", - "eval-id": 1 - } -] \ No newline at end of file diff --git a/work_dirs/aria/Aria-text/result.json b/work_dirs/aria/Aria-text/result.json index 616d6528..06912f2d 100644 --- a/work_dirs/aria/Aria-text/result.json +++ b/work_dirs/aria/Aria-text/result.json @@ -14,7 +14,7 @@ }, { "role": "assistant", - "response": "The result of $1 + 1$ is $2$.<|im_end|>" + "response": "The result of 1 plus 1 is 2.<|im_end|>" } ], "comment": "Test 'no media' modality.", diff --git a/work_dirs/glm-4d1v/GLM-4.1V-9B-Base-single-video-start/result.json b/work_dirs/glm-4d1v/GLM-4.1V-9B-Base-single-video-start/result.json new file mode 100644 index 00000000..b8f7aa5c --- /dev/null +++ b/work_dirs/glm-4d1v/GLM-4.1V-9B-Base-single-video-start/result.json @@ -0,0 +1,94 @@ +[ + { + "id": 0, + "media": [ + "truck.png" + ], + "messages": [ + { + "role": "user", + "question": " Please provide a detailed description of the contents shown in the image.", + "answer": "", + "options": {}, + "choices": [], + "prompt": " Please provide a detailed description of the contents shown in the image.", + "hint": "" + }, + { + "role": "assistant", + "response": "The image displays a GMC truck, specifically a model that appears to be a mid-size pickup, possibly a Canyon or a Sierra. The truck is depicted in motion, as evidenced by the dust being kicked up from its tires. The vehicle is painted in a dark color, likely a shade of gray or dark blue" + } + ], + "comment": "Test 'single image start' modality, 'single video start' modality might be able to handle this, if not, record it in model implement sheet.", + "eval-id": 0 + }, + { + "id": 1, + "media": [ + "boat.png" + ], + "messages": [ + { + "role": "user", + "question": " Please provide a detailed description of the contents shown in the image.", + "answer": "", + "options": {}, + "choices": [], + "prompt": " Please provide a detailed description of the contents shown in the image.", + "hint": "" + }, + { + "role": "assistant", + "response": "The image displays a large, white yacht or motor yacht moving at speed across a body of water, likely the ocean or a large lake. The yacht is the central focus. It has multiple decks, with the main deck featuring a large windshield and windows. The superstructure is white, and the hull is also" + } + ], + "comment": "Test 'single image start' modality, 'single video start' modality might be able to handle this, if not, record it in model implement sheet.", + "eval-id": 1 + }, + { + "id": 2, + "media": [ + "cat.mp4" + ], + "messages": [ + { + "role": "user", + "question": "