diff --git a/docs/superpowers/plans/2026-06-28-android-stabilize-aggregate.md b/docs/superpowers/plans/2026-06-28-android-stabilize-aggregate.md new file mode 100644 index 0000000..e69b4df --- /dev/null +++ b/docs/superpowers/plans/2026-06-28-android-stabilize-aggregate.md @@ -0,0 +1,1402 @@ +# Android perf-runner preflight 安定化 + multi-run median 集計 Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** `tools/rosettadds-perf-runner` に `--stabilize-device` (軽量 preflight)、`--repeat N` (multi-run)、`--aggregate median` (集計) の 3 フラグを追加し、Android 計測の run-to-run variance を縮小する。 + +**Architecture:** 既存 `RunScenario` を multi-run ループで囲み、各 scenario 前に `DeviceStabilizer.StabilizeAsync` で WiFi 切断→再接続 + screen on wakelock + host 接続待機を実施。各 run は `repeat-XX//` に保存、`RunAggregator` が N 個の `metrics.ndjson` を median 集計して `aggregate.json` + manifest.json に追記。TDD で全 unit test を先に書き、最小実装で通す。 + +**Tech Stack:** C# (.NET 8.0), xUnit, FluentAssertions, System.Text.Json, adb + +**Design doc:** `docs/superpowers/specs/2026-06-28-android-stabilize-aggregate-design.md` + +--- + +## 変更 / 作成ファイル一覧 + +| 操作 | パス | 役割 | +| --- | --- | --- | +| Modify | `tools/rosettadds-perf-runner/RunnerOptions.cs` | 3 フラグ追加 + AggregateKind enum | +| Modify | `tools/rosettadds-perf-runner/Program.cs` | scenario loop に stabilize + multi-run 統合 | +| Modify | `tools/rosettadds-perf-runner/ArtifactManifest.cs` | ScenarioManifest に Aggregate プロパティ追加 | +| Create | `tools/rosettadds-perf-runner/DeviceStabilizer.cs` | 軽量 preflight (wakelock + WiFi recycle + ping) | +| Create | `tools/rosettadds-perf-runner/MetricsParser.cs` | NDJSON → measure_done event metrics 抽出 | +| Create | `tools/rosettadds-perf-runner/RunAggregator.cs` | N runs の metrics を median 集計 | +| Modify | `tools/rosettadds-perf-runner.Tests/RunnerOptionsTests.cs` | 新フラグ parse テスト追記 | +| Modify | `tools/rosettadds-perf-runner.Tests/ProgramTests.cs` | multi-run / stabilize integration test | +| Create | `tools/rosettadds-perf-runner.Tests/DeviceStabilizerTests.cs` | preflight の unit test | +| Create | `tools/rosettadds-perf-runner.Tests/MetricsParserTests.cs` | NDJSON parse の unit test | +| Create | `tools/rosettadds-perf-runner.Tests/RunAggregatorTests.cs` | median 集計の unit test | + +Unity / RTPS / DDS / helper のコードは触らない。 + +--- + +## Task 1: `RunnerOptions` に 3 フラグ + `AggregateKind` enum を追加 + +**Files:** +- Modify: `tools/rosettadds-perf-runner/RunnerOptions.cs` + +- [ ] **Step 1.1: 失敗するテストを追加** + +`tools/rosettadds-perf-runner.Tests/RunnerOptionsTests.cs` の末尾に以下を追加: + +```csharp +[Fact] +public void StabilizeDevice_既定値は_false() +{ + var options = RunnerOptions.Parse(Array.Empty()); + options.StabilizeDevice.Should().BeFalse(); +} + +[Fact] +public void StabilizeDevice_指定が_保持される() +{ + var options = RunnerOptions.Parse(new[] { "--stabilize-device" }); + options.StabilizeDevice.Should().BeTrue(); +} + +[Fact] +public void Repeat_既定値は_1() +{ + var options = RunnerOptions.Parse(Array.Empty()); + options.Repeat.Should().Be(1); +} + +[Fact] +public void Repeat_5_が_保持される() +{ + var options = RunnerOptions.Parse(new[] { "--repeat", "5" }); + options.Repeat.Should().Be(5); +} + +[Fact] +public void Repeat_0_は_例外() +{ + var act = () => RunnerOptions.Parse(new[] { "--repeat", "0" }); + act.Should().Throw() + .WithMessage("*--repeat*positive integer*"); +} + +[Fact] +public void Repeat_負値は_例外() +{ + var act = () => RunnerOptions.Parse(new[] { "--repeat", "-1" }); + act.Should().Throw(); +} + +[Fact] +public void Aggregate_既定値は_median() +{ + var options = RunnerOptions.Parse(Array.Empty()); + options.Aggregate.Should().Be(AggregateKind.Median); +} + +[Fact] +public void Aggregate_median_を_受理する() +{ + var options = RunnerOptions.Parse(new[] { "--aggregate", "median" }); + options.Aggregate.Should().Be(AggregateKind.Median); +} + +[Fact] +public void Aggregate_未知値は_例外() +{ + var act = () => RunnerOptions.Parse(new[] { "--aggregate", "stddev" }); + act.Should().Throw() + .WithMessage("*--aggregate*median*"); +} +``` + +- [ ] **Step 1.2: テスト実行 → FAIL 確認** + +```bash +dotnet test tools/rosettadds-perf-runner.Tests --filter "FullyQualifiedName~RunnerOptionsTests" -v minimal +``` + +期待: `StabilizeDevice_既定値は_false` 等 9 件でコンパイルエラー (CS1061) または FAIL。 + +- [ ] **Step 1.3: `RunnerOptions` の修正** + +`tools/rosettadds-perf-runner/RunnerOptions.cs` を以下のように修正する: + +1. クラス冒頭に `enum AggregateKind { Median }` を追加 +2. プロパティ追加 (既存の `bool Help` の下あたり): + ```csharp + internal bool StabilizeDevice { get; private set; } + internal int Repeat { get; private set; } = 1; + internal AggregateKind Aggregate { get; private set; } = AggregateKind.Median; + ``` +3. `Parse` メソッドの switch 文に以下を追加 (`case "--android-activity":` の下): + ```csharp + case "--stabilize-device": + options.StabilizeDevice = true; + break; + case "--repeat": + options.Repeat = ParsePositiveInt(RequireValue(args, ref i, arg), arg); + break; + case "--aggregate": + { + string value = RequireValue(args, ref i, arg); + options.Aggregate = value switch + { + "median" => AggregateKind.Median, + _ => throw new ArgumentException("--aggregate must be median"), + }; + } + break; + ``` +4. `PrintHelp` に 3 行追加 (AndroidDevice 行の下): + ```csharp + output.WriteLine(" --stabilize-device 計測前 Android device 状態を安定化 (WiFi 再接続 + wakelock + ping)"); + output.WriteLine(" --repeat 各 scenario を N 回連続 run (default 1, median 集計対象)"); + output.WriteLine(" --aggregate multi-run 時の集計方法 (default median)"); + ``` + +- [ ] **Step 1.4: テスト実行 → PASS 確認** + +```bash +dotnet test tools/rosettadds-perf-runner.Tests --filter "FullyQualifiedName~RunnerOptionsTests" -v minimal +``` + +期待: `RunnerOptionsTests` の全 19 件 (既存 10 + 新規 9) PASS。 + +- [ ] **Step 1.5: commit** + +```bash +git add tools/rosettadds-perf-runner/RunnerOptions.cs tools/rosettadds-perf-runner.Tests/RunnerOptionsTests.cs +git commit -m "feat(perf-runner): stabilize-device / repeat / aggregate フラグを追加" +``` + +--- + +## Task 2: `DeviceStabilizer` クラス新設 (Android 軽量 preflight) + +**Files:** +- Create: `tools/rosettadds-perf-runner/DeviceStabilizer.cs` +- Create: `tools/rosettadds-perf-runner.Tests/DeviceStabilizerTests.cs` + +- [ ] **Step 2.1: 失敗するテストを作成** + +`tools/rosettadds-perf-runner.Tests/DeviceStabilizerTests.cs` を新規作成: + +```csharp +using System; +using System.Threading; +using System.Threading.Tasks; +using FluentAssertions; +using ROSettaDDS.PerfRunner; +using ROSettaDDS.PerfRunner.Tests.Fakes; +using Xunit; + +namespace ROSettaDDS.PerfRunner.Tests; + +public class DeviceStabilizerTests +{ + [Fact] + public async Task Stabilize_は_wakelock_wifi_recycle_ping_を順に呼ぶ() + { + var fake = new FakeAdbClient(); + var client = new AdbClient(fake, serial: "DEV"); + var stabilizer = new AndroidDeviceStabilizer( + client, hostForPing: "192.168.0.20"); + + await stabilizer.StabilizeAsync(TimeSpan.FromSeconds(5), CancellationToken.None); + + // 1: svc power stayon + // 2: svc wifi disable + // 3: svc wifi enable + // 4: ping -c 5 -W 2 host + fake.Calls.Should().HaveCount(4); + fake.Calls[0].Should().Be("adb -s DEV shell svc power stayon true"); + fake.Calls[1].Should().Be("adb -s DEV shell svc wifi disable"); + fake.Calls[2].Should().Be("adb -s DEV shell svc wifi enable"); + fake.Calls[3].Should().StartWith("adb -s DEV shell ping -c 5 -W 2 192.168.0.20"); + } + + [Fact] + public async Task Stabilize_は_ping_成功で完了する() + { + var fake = new FakeAdbClient(); + // ping は exit 0 が返れば成功とみなす + var client = new AdbClient(fake, serial: "DEV"); + var stabilizer = new AndroidDeviceStabilizer(client, "192.168.0.20"); + + Func act = () => stabilizer.StabilizeAsync(TimeSpan.FromSeconds(5), CancellationToken.None); + await act.Should().NotThrowAsync(); + } + + [Fact] + public async Task Stabilize_は_ping_失敗時にタイムアウト例外を投げる() + { + var fake = new FakeAdbClient { ExitCodeOverride = 1, StderrOverride = "ping: network unreachable" }; + var client = new AdbClient(fake, serial: "DEV"); + var stabilizer = new AndroidDeviceStabilizer(client, "192.168.0.20"); + + Func act = () => stabilizer.StabilizeAsync(TimeSpan.FromMilliseconds(200), CancellationToken.None); + await act.Should().ThrowAsync() + .WithMessage("*ping*"); + } + + [Fact] + public async Task DesktopStabilizer_は_no_op() + { + var fake = new FakeAdbClient(); + var client = new AdbClient(fake, serial: "DEV"); + var stabilizer = new DesktopDeviceStabilizer(); + + await stabilizer.StabilizeAsync(TimeSpan.FromSeconds(5), CancellationToken.None); + + fake.Calls.Should().BeEmpty(); + } +} +``` + +- [ ] **Step 2.2: テスト実行 → FAIL 確認** + +```bash +dotnet test tools/rosettadds-perf-runner.Tests --filter "FullyQualifiedName~DeviceStabilizerTests" -v minimal +``` + +期待: コンパイルエラー (`AndroidDeviceStabilizer`, `DesktopDeviceStabilizer` 未定義)。 + +- [ ] **Step 2.3: `DeviceStabilizer.cs` 実装** + +`tools/rosettadds-perf-runner/DeviceStabilizer.cs` を新規作成: + +```csharp +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; + +namespace ROSettaDDS.PerfRunner; + +internal interface IDeviceStabilizer +{ + Task StabilizeAsync(TimeSpan timeout, CancellationToken ct); +} + +internal sealed class AndroidDeviceStabilizer : IDeviceStabilizer +{ + private readonly AdbClient _adb; + private readonly string _hostForPing; + + public AndroidDeviceStabilizer(AdbClient adb, string hostForPing) + { + _adb = adb; + _hostForPing = hostForPing; + } + + public async Task StabilizeAsync(TimeSpan timeout, CancellationToken ct) + { + // 1) screen on wakelock + var r1 = await _adb.RunAsync( + $"adb -s {_adb.Serial} shell svc power stayon true", ct); + if (r1.ExitCode != 0) + { + throw new InvalidOperationException( + $"svc power stayon failed (exit={r1.ExitCode}): {r1.Stderr.Trim()}"); + } + + // 2) WiFi recycle: disable → enable + var r2 = await _adb.RunAsync( + $"adb -s {_adb.Serial} shell svc wifi disable", ct); + if (r2.ExitCode != 0) + { + throw new InvalidOperationException( + $"svc wifi disable failed (exit={r2.ExitCode}): {r2.Stderr.Trim()}"); + } + // 1 秒待機 (WiFi 切断完了待ち) + await Task.Delay(TimeSpan.FromSeconds(1), ct); + var r3 = await _adb.RunAsync( + $"adb -s {_adb.Serial} shell svc wifi enable", ct); + if (r3.ExitCode != 0) + { + throw new InvalidOperationException( + $"svc wifi enable failed (exit={r3.ExitCode}): {r3.Stderr.Trim()}"); + } + + // 3) host への接続確認 (最大 timeout までリトライ) + DateTimeOffset deadline = DateTimeOffset.UtcNow + timeout; + string pingCmd = $"adb -s {_adb.Serial} shell ping -c 5 -W 2 {_hostForPing}"; + while (DateTimeOffset.UtcNow < deadline) + { + ct.ThrowIfCancellationRequested(); + var rp = await _adb.RunAsync(pingCmd, ct); + if (rp.ExitCode == 0) + { + return; + } + await Task.Delay(TimeSpan.FromMilliseconds(500), ct); + } + throw new TimeoutException( + $"timed out waiting for ping {_hostForPing} to succeed within {timeout}"); + } +} + +internal sealed class DesktopDeviceStabilizer : IDeviceStabilizer +{ + public Task StabilizeAsync(TimeSpan timeout, CancellationToken ct) => Task.CompletedTask; +} +``` + +- [ ] **Step 2.4: テスト実行 → PASS 確認** + +```bash +dotnet test tools/rosettadds-perf-runner.Tests --filter "FullyQualifiedName~DeviceStabilizerTests" -v minimal +``` + +期待: 4 件全 PASS。 + +- [ ] **Step 2.5: commit** + +```bash +git add tools/rosettadds-perf-runner/DeviceStabilizer.cs tools/rosettadds-perf-runner.Tests/DeviceStabilizerTests.cs +git commit -m "feat(perf-runner): DeviceStabilizer (Android 軽量 preflight) を追加" +``` + +--- + +## Task 3: `MetricsParser` クラス新設 (NDJSON → measure_done metrics) + +**Files:** +- Create: `tools/rosettadds-perf-runner/MetricsParser.cs` +- Create: `tools/rosettadds-perf-runner.Tests/MetricsParserTests.cs` + +- [ ] **Step 3.1: 失敗するテストを作成** + +`tools/rosettadds-perf-runner.Tests/MetricsParserTests.cs` を新規作成: + +```csharp +using System; +using System.IO; +using FluentAssertions; +using ROSettaDDS.PerfRunner; +using Xunit; + +namespace ROSettaDDS.PerfRunner.Tests; + +public class MetricsParserTests +{ + [Fact] + public void ParseMeasureDone_は_measure_done_event_を抽出する() + { + string path = Path.Combine(Path.GetTempPath(), "metrics-parser-test.ndjson"); + File.WriteAllText(path, + "{\"event\":\"start\",\"scenario\":\"x\"}\n" + + "{\"event\":\"ready\"}\n" + + "{\"event\":\"measure_done\",\"messages_per_second\":1234.5,\"elapsed_ms\":50.0,\"received\":500,\"main_thread_time_ns_last\":82622}\n" + + "{\"event\":\"done\"}\n"); + try + { + var result = MetricsParser.ParseMeasureDone(path); + result.Should().NotBeNull(); + result!.MessagesPerSecond.Should().Be(1234.5); + result.ElapsedMs.Should().Be(50.0); + result.Received.Should().Be(500); + result.MainThreadTimeNsLast.Should().Be(82622); + } + finally + { + if (File.Exists(path)) File.Delete(path); + } + } + + [Fact] + public void ParseMeasureDone_は_空ファイルで_null() + { + string path = Path.Combine(Path.GetTempPath(), "metrics-parser-test-empty.ndjson"); + File.WriteAllText(path, string.Empty); + try + { + var result = MetricsParser.ParseMeasureDone(path); + result.Should().BeNull(); + } + finally + { + if (File.Exists(path)) File.Delete(path); + } + } + + [Fact] + public void ParseMeasureDone_は_measure_done_不在で_null() + { + string path = Path.Combine(Path.GetTempPath(), "metrics-parser-test-noevent.ndjson"); + File.WriteAllText(path, + "{\"event\":\"start\"}\n" + + "{\"event\":\"ready\"}\n" + + "{\"event\":\"done\"}\n"); + try + { + var result = MetricsParser.ParseMeasureDone(path); + result.Should().BeNull(); + } + finally + { + if (File.Exists(path)) File.Delete(path); + } + } + + [Fact] + public void ParseMeasureDone_は_不正_JSON_行をスキップする() + { + string path = Path.Combine(Path.GetTempPath(), "metrics-parser-test-bad.ndjson"); + File.WriteAllText(path, + "garbage line\n" + + "{\"event\":\"measure_done\",\"messages_per_second\":42.0,\"elapsed_ms\":10.0,\"received\":10}\n"); + try + { + var result = MetricsParser.ParseMeasureDone(path); + result.Should().NotBeNull(); + result!.MessagesPerSecond.Should().Be(42.0); + } + finally + { + if (File.Exists(path)) File.Delete(path); + } + } + + [Fact] + public void ParseMeasureDone_は_ファイル不在で_null() + { + var result = MetricsParser.ParseMeasureDone("/nonexistent/path/metrics.ndjson"); + result.Should().BeNull(); + } +} +``` + +- [ ] **Step 3.2: テスト実行 → FAIL 確認** + +```bash +dotnet test tools/rosettadds-perf-runner.Tests --filter "FullyQualifiedName~MetricsParserTests" -v minimal +``` + +期待: コンパイルエラー (`MetricsParser` 未定義)。 + +- [ ] **Step 3.3: `MetricsParser.cs` 実装** + +`tools/rosettadds-perf-runner/MetricsParser.cs` を新規作成: + +```csharp +using System; +using System.IO; +using System.Text.Json; + +namespace ROSettaDDS.PerfRunner; + +internal sealed class MeasureDoneMetrics +{ + public double MessagesPerSecond { get; set; } + public double ElapsedMs { get; set; } + public long Received { get; set; } + public long MainThreadTimeNsLast { get; set; } + public long GcReservedMemoryBytesLast { get; set; } + public long GcUsedMemoryBytesLast { get; set; } + public long SystemUsedMemoryBytesLast { get; set; } + public long SerializedBytesPerMessage { get; set; } +} + +internal static class MetricsParser +{ + public static MeasureDoneMetrics? ParseMeasureDone(string path) + { + if (!File.Exists(path)) return null; + + MeasureDoneMetrics? latest = null; + foreach (string line in File.ReadAllLines(path)) + { + if (string.IsNullOrWhiteSpace(line)) continue; + try + { + using JsonDocument doc = JsonDocument.Parse(line); + JsonElement root = doc.RootElement; + if (!root.TryGetProperty("event", out JsonElement ev)) continue; + if (ev.GetString() != "measure_done") continue; + latest = new MeasureDoneMetrics + { + MessagesPerSecond = ReadDouble(root, "messages_per_second"), + ElapsedMs = ReadDouble(root, "elapsed_ms"), + Received = ReadLong(root, "received"), + MainThreadTimeNsLast = ReadLong(root, "main_thread_time_ns_last"), + GcReservedMemoryBytesLast = ReadLong(root, "gc_reserved_memory_bytes_last"), + GcUsedMemoryBytesLast = ReadLong(root, "gc_used_memory_bytes_last"), + SystemUsedMemoryBytesLast = ReadLong(root, "system_used_memory_bytes_last"), + SerializedBytesPerMessage = ReadLong(root, "serialized_bytes_per_message"), + }; + } + catch (JsonException) + { + // 不正 JSON 行はスキップ + continue; + } + } + return latest; + } + + private static double ReadDouble(JsonElement root, string name) + { + if (root.TryGetProperty(name, out JsonElement v) && v.ValueKind == JsonValueKind.Number) + { + return v.GetDouble(); + } + return 0.0; + } + + private static long ReadLong(JsonElement root, string name) + { + if (root.TryGetProperty(name, out JsonElement v) && v.ValueKind == JsonValueKind.Number) + { + return v.GetInt64(); + } + return 0; + } +} +``` + +- [ ] **Step 3.4: テスト実行 → PASS 確認** + +```bash +dotnet test tools/rosettadds-perf-runner.Tests --filter "FullyQualifiedName~MetricsParserTests" -v minimal +``` + +期待: 5 件全 PASS。 + +- [ ] **Step 3.5: commit** + +```bash +git add tools/rosettadds-perf-runner/MetricsParser.cs tools/rosettadds-perf-runner.Tests/MetricsParserTests.cs +git commit -m "feat(perf-runner): MetricsParser (NDJSON → measure_done) を追加" +``` + +--- + +## Task 4: `RunAggregator` クラス新設 (N runs の metrics を median 集計) + +**Files:** +- Create: `tools/rosettadds-perf-runner/RunAggregator.cs` +- Create: `tools/rosettadds-perf-runner.Tests/RunAggregatorTests.cs` + +- [ ] **Step 4.1: 失敗するテストを作成** + +`tools/rosettadds-perf-runner.Tests/RunAggregatorTests.cs` を新規作成: + +```csharp +using System; +using System.Collections.Generic; +using System.IO; +using FluentAssertions; +using ROSettaDDS.PerfRunner; +using Xunit; + +namespace ROSettaDDS.PerfRunner.Tests; + +public class RunAggregatorTests +{ + [Fact] + public void Aggregate_は_1_run_でその値を返す() + { + var result = new[] { 100.0 }; + RunAggregator.Median(result).Should().Be(100.0); + } + + [Fact] + public void Aggregate_Median_は_奇数個で中央値() + { + RunAggregator.Median(new[] { 1.0, 5.0, 3.0, 9.0, 7.0 }).Should().Be(5.0); + } + + [Fact] + public void Aggregate_Median_は_偶数個で中央2値平均() + { + RunAggregator.Median(new[] { 1.0, 2.0, 3.0, 4.0 }).Should().Be(2.5); + } + + [Fact] + public void Aggregate_Median_は_空で_0() + { + RunAggregator.Median(Array.Empty()).Should().Be(0.0); + } + + [Fact] + public void Aggregate_Median_は_未ソート入力でも正しい() + { + RunAggregator.Median(new[] { 9.0, 1.0, 5.0, 3.0, 7.0 }).Should().Be(5.0); + } + + [Fact] + public void Aggregate_は_N個のmetrics_pathから_measure_done_metricsを_集計する() + { + string tempDir = Path.Combine(Path.GetTempPath(), "agg-test-" + Guid.NewGuid()); + Directory.CreateDirectory(tempDir); + try + { + // 3 runs の metrics.ndjson を作成 + WriteMeasureDone(Path.Combine(tempDir, "repeat-00", "metrics.ndjson"), + mps: 100, elapsed: 50, received: 500); + WriteMeasureDone(Path.Combine(tempDir, "repeat-01", "metrics.ndjson"), + mps: 200, elapsed: 25, received: 500); + WriteMeasureDone(Path.Combine(tempDir, "repeat-02", "metrics.ndjson"), + mps: 300, elapsed: 16, received: 500); + + // 各 repeat-XX 配下を 1 つの runDir として aggregator に渡す + var runDirs = new[] + { + Path.Combine(tempDir, "repeat-00"), + Path.Combine(tempDir, "repeat-01"), + Path.Combine(tempDir, "repeat-02"), + }; + var aggregate = RunAggregator.Aggregate(runDirs, AggregateKind.Median); + aggregate.Should().NotBeNull(); + aggregate!.MessagesPerSecond.Should().Be(200.0); + aggregate.ElapsedMs.Should().Be(25.0); + aggregate.Received.Should().Be(500); + aggregate.RunCount.Should().Be(3); + } + finally + { + if (Directory.Exists(tempDir)) Directory.Delete(tempDir, recursive: true); + } + } + + [Fact] + public void Aggregate_は_metrics_欠損_runをスキップして_残りで集計() + { + string tempDir = Path.Combine(Path.GetTempPath(), "agg-test-missing-" + Guid.NewGuid()); + Directory.CreateDirectory(tempDir); + try + { + WriteMeasureDone(Path.Combine(tempDir, "repeat-00", "metrics.ndjson"), + mps: 100, elapsed: 50, received: 500); + // repeat-01 は metrics.ndjson なし + WriteMeasureDone(Path.Combine(tempDir, "repeat-02", "metrics.ndjson"), + mps: 300, elapsed: 16, received: 500); + + var runDirs = new[] + { + Path.Combine(tempDir, "repeat-00"), + Path.Combine(tempDir, "repeat-01"), + Path.Combine(tempDir, "repeat-02"), + }; + var aggregate = RunAggregator.Aggregate(runDirs, AggregateKind.Median); + aggregate.Should().NotBeNull(); + aggregate!.MessagesPerSecond.Should().Be(200.0); + aggregate.RunCount.Should().Be(2); + } + finally + { + if (Directory.Exists(tempDir)) Directory.Delete(tempDir, recursive: true); + } + } + + [Fact] + public void Aggregate_は_全_run_で_metrics_なし_で_null() + { + string tempDir = Path.Combine(Path.GetTempPath(), "agg-test-empty-" + Guid.NewGuid()); + Directory.CreateDirectory(tempDir); + try + { + var runDirs = new[] + { + Path.Combine(tempDir, "repeat-00"), + Path.Combine(tempDir, "repeat-01"), + }; + var aggregate = RunAggregator.Aggregate(runDirs, AggregateKind.Median); + aggregate.Should().BeNull(); + } + finally + { + if (Directory.Exists(tempDir)) Directory.Delete(tempDir, recursive: true); + } + } + + private static void WriteMeasureDone(string path, double mps, double elapsed, long received) + { + Directory.CreateDirectory(Path.GetDirectoryName(path)!); + File.WriteAllText(path, + "{\"event\":\"start\"}\n" + + $"{{\"event\":\"measure_done\",\"messages_per_second\":{mps},\"elapsed_ms\":{elapsed},\"received\":{received}}}\n" + + "{\"event\":\"done\"}\n"); + } +} +``` + +- [ ] **Step 4.2: テスト実行 → FAIL 確認** + +```bash +dotnet test tools/rosettadds-perf-runner.Tests --filter "FullyQualifiedName~RunAggregatorTests" -v minimal +``` + +期待: コンパイルエラー (`RunAggregator` 未定義)。 + +- [ ] **Step 4.3: `RunAggregator.cs` 実装** + +`tools/rosettadds-perf-runner/RunAggregator.cs` を新規作成: + +```csharp +using System; +using System.Collections.Generic; +using System.IO; +using System.Text.Json; + +namespace ROSettaDDS.PerfRunner; + +internal sealed class AggregateMetrics +{ + public int RunCount { get; set; } + public double MessagesPerSecond { get; set; } + public double ElapsedMs { get; set; } + public long Received { get; set; } + public long MainThreadTimeNsLast { get; set; } + public long GcReservedMemoryBytesLast { get; set; } + public long GcUsedMemoryBytesLast { get; set; } + public long SystemUsedMemoryBytesLast { get; set; } + public long SerializedBytesPerMessage { get; set; } +} + +internal static class RunAggregator +{ + public static double Median(IReadOnlyList values) + { + if (values.Count == 0) return 0.0; + var sorted = new double[values.Count]; + for (int i = 0; i < values.Count; i++) sorted[i] = values[i]; + Array.Sort(sorted); + int mid = sorted.Length / 2; + if (sorted.Length % 2 == 1) + { + return sorted[mid]; + } + return (sorted[mid - 1] + sorted[mid]) / 2.0; + } + + public static long MedianLong(IReadOnlyList values) + { + if (values.Count == 0) return 0; + var sorted = new long[values.Count]; + for (int i = 0; i < values.Count; i++) sorted[i] = values[i]; + Array.Sort(sorted); + int mid = sorted.Length / 2; + if (sorted.Length % 2 == 1) + { + return sorted[mid]; + } + return (sorted[mid - 1] + sorted[mid]) / 2; + } + + public static AggregateMetrics? Aggregate( + IReadOnlyList runDirs, + AggregateKind kind) + { + if (kind != AggregateKind.Median) + { + throw new ArgumentException("only median is currently supported"); + } + var mpsList = new List(); + var elapsedList = new List(); + var receivedList = new List(); + var mainThreadList = new List(); + var gcReservedList = new List(); + var gcUsedList = new List(); + var sysUsedList = new List(); + var serBytesList = new List(); + + foreach (string dir in runDirs) + { + string metricsPath = Path.Combine(dir, "metrics.ndjson"); + MeasureDoneMetrics? m = MetricsParser.ParseMeasureDone(metricsPath); + if (m == null) continue; + mpsList.Add(m.MessagesPerSecond); + elapsedList.Add(m.ElapsedMs); + receivedList.Add(m.Received); + mainThreadList.Add(m.MainThreadTimeNsLast); + gcReservedList.Add(m.GcReservedMemoryBytesLast); + gcUsedList.Add(m.GcUsedMemoryBytesLast); + sysUsedList.Add(m.SystemUsedMemoryBytesLast); + serBytesList.Add(m.SerializedBytesPerMessage); + } + + if (mpsList.Count == 0) return null; + + return new AggregateMetrics + { + RunCount = mpsList.Count, + MessagesPerSecond = Median(mpsList), + ElapsedMs = Median(elapsedList), + Received = MedianLong(receivedList), + MainThreadTimeNsLast = MedianLong(mainThreadList), + GcReservedMemoryBytesLast = MedianLong(gcReservedList), + GcUsedMemoryBytesLast = MedianLong(gcUsedList), + SystemUsedMemoryBytesLast = MedianLong(sysUsedList), + SerializedBytesPerMessage = MedianLong(serBytesList), + }; + } + + public static void Save(string path, AggregateMetrics metrics) + { + var options = new JsonSerializerOptions { WriteIndented = true }; + File.WriteAllText(path, JsonSerializer.Serialize(metrics, options)); + } +} +``` + +- [ ] **Step 4.4: テスト実行 → PASS 確認** + +```bash +dotnet test tools/rosettadds-perf-runner.Tests --filter "FullyQualifiedName~RunAggregatorTests" -v minimal +``` + +期待: 8 件全 PASS。 + +- [ ] **Step 4.5: commit** + +```bash +git add tools/rosettadds-perf-runner/RunAggregator.cs tools/rosettadds-perf-runner.Tests/RunAggregatorTests.cs +git commit -m "feat(perf-runner): RunAggregator (median 集計) を追加" +``` + +--- + +## Task 5: `ArtifactManifest.ScenarioManifest` に `Aggregate` プロパティ追加 + +**Files:** +- Modify: `tools/rosettadds-perf-runner/ArtifactManifest.cs` + +- [ ] **Step 5.1: `Aggregate` プロパティ追加** + +`tools/rosettadds-perf-runner/ArtifactManifest.cs` の `ScenarioManifest` クラスを以下に修正 +(既存プロパティの後に `Aggregate` と `RepeatCount` を追加): + +```csharp +internal sealed class ScenarioManifest +{ + public string Name { get; set; } = ""; + public string Direction { get; set; } = ""; + public string MetricsPath { get; set; } = ""; + public string ProfilerPath { get; set; } = ""; + public string PlayerLogPath { get; set; } = ""; + public string HelperStdoutPath { get; set; } = ""; + public string HelperStderrPath { get; set; } = ""; + public int PlayerExitCode { get; set; } + public int HelperExitCode { get; set; } + public int RepeatCount { get; set; } = 1; + public string? AggregatePath { get; set; } + public AggregateMetrics? Aggregate { get; set; } +} +``` + +- [ ] **Step 5.2: ビルド確認** + +```bash +dotnet build tools/rosettadds-perf-runner -c Release +``` + +期待: 0 errors / 0 warnings。 + +- [ ] **Step 5.3: 既存テスト回帰確認** + +```bash +dotnet test tools/rosettadds-perf-runner.Tests -v minimal +``` + +期待: 既存全テスト PASS (ArtifactManifest 周りで破壊していないこと)。 + +- [ ] **Step 5.4: commit** + +```bash +git add tools/rosettadds-perf-runner/ArtifactManifest.cs +git commit -m "feat(perf-runner): ScenarioManifest に Aggregate / RepeatCount プロパティを追加" +``` + +--- + +## Task 6: `Program.MainAsync` を multi-run + stabilize 対応に改修 + +**Files:** +- Modify: `tools/rosettadds-perf-runner/Program.cs` + +- [ ] **Step 6.1: 統合テストを追加** + +`tools/rosettadds-perf-runner.Tests/ProgramTests.cs` の末尾に以下を追加 +(既存 `ProgramTests.cs` の構造に従い、必要なら using を追加): + +```csharp +using System; +using System.IO; +using System.Threading.Tasks; +using FluentAssertions; +using ROSettaDDS.PerfRunner; +using ROSettaDDS.PerfRunner.Tests.Fakes; +using Xunit; + +namespace ROSettaDDS.PerfRunner.Tests; + +public class ProgramMultiRunTests +{ + [Fact] + public async Task MainAsync_repeat_3_で_各_scenario_を_3_run_する() + { + string runDir = Path.Combine(Path.GetTempPath(), "prog-multirun-test-" + Guid.NewGuid()); + try + { + // 検証: --repeat 3 指定で metrics.ndjson が 3 個作成される + // (FakeProcessDriver / FakeAdbClient が必要、本テストは最小実装) + // 詳細は Step 6.4 で実装確認 + } + finally + { + if (Directory.Exists(runDir)) Directory.Delete(runDir, recursive: true); + } + } +} +``` + +注意: 上のテストは Task 6 内で FakeProcessDriver / FakeAdbClient を使った完全形に +置き換える (Step 6.4 で差し替え)。 + +- [ ] **Step 6.2: `Program.cs` の改修** + +`tools/rosettadds-perf-runner/Program.cs` の `MainAsync` を以下のように修正する: + +1. 既存の `IDeviceStabilizer` 取得ロジック追加 (RunScenario ループの前): + + ```csharp + IDeviceStabilizer stabilizer = options.BuildTarget == "Android" + ? new AndroidDeviceStabilizer( + new AdbClient(new RealAdbCommandSink(options.Adb), options.AndroidDevice + ?? throw new InvalidOperationException("--android-device is required for --stabilize-device on Android")), + hostForPing: "192.168.0.20" // 固定値。設定可能化は別 PR (out of scope) + ) + : new DesktopDeviceStabilizer(); + + if (options.StabilizeDevice && options.BuildTarget != "Android") + { + Console.Error.WriteLine($"[warn] --stabilize-device is Android-only, ignored for {options.BuildTarget}"); + } + ``` + +2. `for (int i = 0; i < scenarios.Count; i++)` の中身を multi-run 対応に: + + ```csharp + for (int i = 0; i < scenarios.Count; i++) + { + PerfScenario scenario = scenarios[i]; + + // multi-run 用の runDir 群を準備 + var scenarioRunDirs = new List(options.Repeat); + for (int r = 0; r < options.Repeat; r++) + { + string runDir = Path.Combine(runDir, scenario.Name, $"repeat-{r:D2}"); + scenarioRunDirs.Add(runDir); + } + + ScenarioManifest scenarioManifest; + for (int r = 0; r < options.Repeat; r++) + { + if (options.StabilizeDevice && options.BuildTarget == "Android") + { + try + { + await stabilizer.StabilizeAsync(TimeSpan.FromSeconds(30), CancellationToken.None) + .ConfigureAwait(false); + } + catch (Exception ex) + { + Console.Error.WriteLine($"[warn] device stabilization failed (run {r}): {ex.Message}"); + } + } + scenarioManifest = await RunScenario( + root, + helper, + playerExecutable, + scenarioRunDirs[r], + scenario, + options, + domainBase + i).ConfigureAwait(false); + scenarioManifest.Name = scenario.Name; // multi-run で上書きされるのを防ぐ + scenarioManifest.RepeatCount = options.Repeat; + if (r == 0) + { + manifest.Scenarios.Add(scenarioManifest); + } + else + { + // 2 回目以降は最後の scenario manifest を上書き + manifest.Scenarios[manifest.Scenarios.Count - 1] = scenarioManifest; + } + if (scenarioManifest.PlayerExitCode != 0 || scenarioManifest.HelperExitCode != 0) + { + failed = true; + } + } + + // aggregate (N > 1 の時のみ) + if (options.Repeat > 1) + { + var aggregate = RunAggregator.Aggregate(scenarioRunDirs, options.Aggregate); + if (aggregate != null) + { + string aggregatePath = Path.Combine(runDir, scenario.Name, "aggregate.json"); + RunAggregator.Save(aggregatePath, aggregate); + manifest.Scenarios[manifest.Scenarios.Count - 1].AggregatePath = aggregatePath; + manifest.Scenarios[manifest.Scenarios.Count - 1].Aggregate = aggregate; + } + } + + manifest.Save(Path.Combine(runDir, "manifest.json")); + } + ``` + +3. 既存 `RunScenario` 呼び出しの最初の引数 (`runDir`) を、multi-run では + `scenarioRunDirs[r]` に変更するため、`RunScenario` シグネチャの `runDir` パラメータが + そのまま使える (内部で `Path.Combine(runDir, scenario.Name)` していた箇所を削除)。 + + ただし、既存 `RunScenario` の `string runDir` パラメータは「scenario 親ディレクトリ」 + を期待しているので、呼び出し側で `runDir` (= 計測 runDir) を渡すのは変。 + `RunScenario` 内で `scenarioDir = Path.Combine(runDir, scenario.Name)` していた + ロジックを削除し、`scenarioDir` パラメータを直接受け取る形に変更する: + + `RunScenario` の先頭を: + ```csharp + string scenarioDir = runDir; // 既に scenario 配下のパスが来る + Directory.CreateDirectory(scenarioDir); + ``` + に変更。 + +- [ ] **Step 6.3: ビルド確認** + +```bash +dotnet build tools/rosettadds-perf-runner -c Release +``` + +期待: 0 errors。warning があれば確認して対応。 + +- [ ] **Step 6.4: ProgramTests の差し替え** + +`tools/rosettadds-perf-runner.Tests/ProgramTests.cs` の `ProgramMultiRunTests` を +以下のように差し替え (FakeProcessDriver を使った完全版): + +```csharp +using System; +using System.IO; +using System.Threading.Tasks; +using FluentAssertions; +using ROSettaDDS.PerfRunner; +using ROSettaDDS.PerfRunner.Tests.Fakes; +using Xunit; + +namespace ROSettaDDS.PerfRunner.Tests; + +public class ProgramMultiRunTests +{ + [Fact] + public void MainAsync_既存引数_は_従来通り_動作する() + { + // --repeat 1 デフォルトで aggregate は発生しない + // 既存 ProgramTests のテストが破壊されていないことを確認 + // (本ファイル内の他テストで担保) + } +} +``` + +注意: 既存 `ProgramTests.cs` のテストが全て PASS していれば OK。新規 multi-run / +stabilize の end-to-end テストは FakeProcessDriver / FakeAdbClient 経由での完全 +integration が必要となり、複雑度が増す。**Task 6 では ProgramTests の既存テスト +が破壊されていないことのみ確認**し、end-to-end テストは Task 7 (実機計測) で +担保する。 + +- [ ] **Step 6.5: 全テスト回帰確認** + +```bash +dotnet test tools/rosettadds-perf-runner.Tests -v minimal +``` + +期待: 全テスト (既存 + Task 1-5 で追加した分) PASS。 + +- [ ] **Step 6.6: commit** + +```bash +git add tools/rosettadds-perf-runner/Program.cs tools/rosettadds-perf-runner.Tests/ProgramTests.cs +git commit -m "feat(perf-runner): MainAsync に multi-run + stabilize 統合" +``` + +--- + +## Task 7: 既存 help / バリデーション整合確認 + 最終ビルド + +- [ ] **Step 7.1: `--help` 出力確認** + +```bash +dotnet run --project tools/rosettadds-perf-runner -c Release --no-build -- --help +``` + +期待: `--stabilize-device` / `--repeat` / `--aggregate` の 3 行が help に出力。 + +- [ ] **Step 7.2: バリデーション確認** + +```bash +dotnet run --project tools/rosettadds-perf-runner -c Release --no-build -- --repeat 0 +``` + +期待: `Unhandled exception: --repeat must be a positive integer` で exit 1。 + +```bash +dotnet run --project tools/rosettadds-perf-runner -c Release --no-build -- --aggregate stddev +``` + +期待: `Unhandled exception: --aggregate must be median` で exit 1。 + +- [ ] **Step 7.3: 既存単体フラグの後方互換性確認** + +```bash +dotnet run --project tools/rosettadds-perf-runner -c Release --no-build -- --build-target StandaloneLinux64 --scenario unity-to-ros2-reliable-32 --skip-build --player-build /tmp/ROSettaDDSPerfPlayer --artifacts /tmp/perf-backcompat-check --capture-frames 100 +``` + +期待: 既存 scenario 実行、aggregate.json は作成されない、manifest.json は従来形式 +(`AggregatePath` / `Aggregate` フィールドは null だが JSON には含まれる)。 + +- [ ] **Step 7.4: 全テスト最終確認** + +```bash +dotnet test tools/rosettadds-perf-runner.Tests -c Release -v minimal +``` + +期待: 全テスト PASS。 + +- [ ] **Step 7.5: commit (変更なしの場合スキップ)** + +```bash +git status +``` + +変更があれば commit、なければスキップ。 + +--- + +## Task 8: 実機計測 (Android + Sony XIG04) + +> 注: device 接続必須。device 不在の場合は `--skip-build` + Desktop 計測で代替確認。 + +- [ ] **Step 8.1: device 接続 + 既存 build artifact 確認** + +```bash +adb devices +ls -la /tmp/rosettadds-perf-android.apk +``` + +期待: `5HF6OVWCDECMJZ59 device` 出力。apk 不在なら `dotnet run --project +tools/rosettadds-perf-runner -c Release -- --build-target Android --scenario all +--android-device 5HF6OVWCDECMJZ59 --artifacts /tmp/build-check` で再 build。 + +- [ ] **Step 8.2: stabilize + repeat 5 で 1 scenario 計測 (smoke)** + +```bash +mkdir -p artifacts/perf-android-stabilize +dotnet run --project tools/rosettadds-perf-runner -c Release --no-build -- \ + --build-target Android \ + --scenario unity-to-ros2-best-effort-8192 \ + --android-device 5HF6OVWCDECMJZ59 \ + --player-build /tmp/rosettadds-perf-android.apk \ + --artifacts artifacts/perf-android-stabilize \ + --stabilize-device \ + --repeat 5 \ + --aggregate median \ + --capture-frames 200 +``` + +期待: 計測完走。`artifacts/perf-android-stabilize//unity-to-ros2-best-effort-8192/` +配下に `repeat-00/` 〜 `repeat-04/` の 5 ディレクトリ + `aggregate.json` 1 ファイル ++ `manifest.json` 1 ファイル。 + +- [ ] **Step 8.3: aggregate.json 内容確認** + +```bash +cat artifacts/perf-android-stabilize//unity-to-ros2-best-effort-8192/aggregate.json +``` + +期待: `runCount: 5` と各メトリクス (mps, elapsedMs, received, ...) の median 値。 + +- [ ] **Step 8.4: manifest.json 内容確認** + +```bash +cat artifacts/perf-android-stabilize//manifest.json +``` + +期待: `Scenarios[0].RepeatCount == 5`、`Aggregate` / `AggregatePath` フィールドが存在。 + +- [ ] **Step 8.5: 9 scenario 全部を multi-run で計測 (約 30-60 分)** + +```bash +dotnet run --project tools/rosettadds-perf-runner -c Release --no-build -- \ + --build-target Android \ + --scenario all \ + --android-device 5HF6OVWCDECMJZ59 \ + --player-build /tmp/rosettadds-perf-android.apk \ + --artifacts artifacts/perf-android-stabilize-full \ + --stabilize-device \ + --repeat 5 \ + --aggregate median \ + --capture-frames 200 +``` + +期待: 全 9 scenario 完走、各 scenario 配下に 5 runs + aggregate.json。 + +- [ ] **Step 8.6: artifacts を git commit** + +```bash +git add artifacts/perf-android-stabilize artifacts/perf-android-stabilize-full +git commit -m "chore: stabilize + repeat 5 計測 artifacts (Android 5HF6OVWCDECMJZ59)" +``` + +注意: 計測ログのみコミット。.pcap や大きな binary は .gitignore で除外。 + +--- + +## Task 9: findings doc 作成 + +**Files:** +- Create: `docs/superpowers/specs/2026-06-28-android-stabilize-aggregate-findings.md` + +- [ ] **Step 9.1: 計測結果集計** + +```bash +# 9 scenario の median mps を集計 +for s in artifacts/perf-android-stabilize-full/*/; do + run=$(basename "$s") + for sc in "$s"*/; do + scn=$(basename "$sc") + if [ -f "$sc/aggregate.json" ]; then + mps=$(python3 -c "import json; print(json.load(open('$sc/aggregate.json'))['messagesPerSecond'])") + echo "$run $scn $mps" + fi + done +done +``` + +- [ ] **Step 9.2: 既存 run (20260627-103647) との比較表作成** + +既存 `artifacts/perf-android-all/20260627-103647/manifest.json` (--repeat 1) と +新 `artifacts/perf-android-stabilize-full//manifest.json` (--repeat 5) の +mps を比較する markdown table を作る。 + +- [ ] **Step 9.3: findings doc 作成** + +`docs/superpowers/specs/2026-06-28-android-stabilize-aggregate-findings.md` を +以下のテンプレートで作成: + +```markdown +# Android Stabilize + Multi-run Findings (2026-06-28) + +## サマリ + +`tools/rosettadds-perf-runner` に `--stabilize-device` (軽量 preflight)、 +`--repeat 5` (multi-run)、`--aggregate median` (集計) を追加し、Sony XIG04 +(Xperia 10 III / Android 15) で 9 scenario × 5 runs を計測。既存 run +(2026-06-27 --repeat 1) と比較して run-to-run variance が (n× → m×) に縮小 +したことを確認。 + +## 計測環境 + +- 日時: 2026-06-28 +- HEAD: +- branch: perf/android-stabilize-and-aggregate +- device: Sony XIG04 (Xperia 10 III), Android 15 +- 同 L2 セグメント (host 192.168.0.20 ↔ device 192.168.0.22) + +## 計測結果 + +### 既存 (--repeat 1) vs 新 (--repeat 5 median) 比較 + +| Scenario | 既存 mps | 既存 run-to-run variance | 新 median mps | 新 variance | +|----------|---------:|-------------------------:|--------------:|------------:| +| ... | | | | | + +### preflight ON vs OFF 比較 (任意、smoke 計測で取得した場合) + +(あれば) + +## 結論 + +- preflight + repeat 5 により variance が縮小した / しなかった +- 縮小した場合の主要因 (WiFi 切断による connection pool クリーンアップ、wakelock + による screen off 防止、等) +- 縮小しなかった場合は原因を別途分析 + +## 残存ボトルネック / next action + +1. 既存 findings の他 next action への取り組み状況 +2. preflight 拡張 (device リブート、permission 許可) の必要性 +3. helper 側詳細解析 +4. ... + +## 計測 artifact + +- `artifacts/perf-android-stabilize//` (smoke 1 scenario × 5 runs) +- `artifacts/perf-android-stabilize-full//` (full 9 scenario × 5 runs) +- 比較元: `artifacts/perf-android-all/20260627-103647/` (既存 --repeat 1) +``` + +- [ ] **Step 9.4: commit** + +```bash +git add docs/superpowers/specs/2026-06-28-android-stabilize-aggregate-findings.md +git commit -m "docs(specs): Android stabilize + multi-run 計測結果と findings を追加" +``` + +--- + +## Task 10: 検証 + PR + +- [ ] **Step 10.1: Validation チェックリスト実行** + +design doc の Validation 完了条件を 1 つずつ確認: + +- [ ] `--stabilize-device --repeat 5 --aggregate median` で全 9 scenario 完走 +- [ ] 既存 run (--repeat 1) と新 run (--repeat 5) で manifest.json の + scenario 数が一致 +- [ ] aggregate.json に全 metric の median 値が出力される +- [ ] preflight ON vs OFF で run-to-run variance が縮小することを確認 +- [ ] 既存 `dotnet run --project tools/rosettadds-perf-runner` 単独利用の + シナリオ (--repeat 1 デフォルト) は無変更動作 + +- [ ] **Step 10.2: 既存 spec doc との整合性確認** + +`docs/superpowers/specs/2026-06-27-android-bottleneck-investigation.md` の +「Next action 2: Android run-to-run variance 10× の安定化 (B')」が本 findings +で更新されたか確認。 + +- [ ] **Step 10.3: ブランチ push** + +```bash +git push origin perf/android-stabilize-and-aggregate +``` + +- [ ] **Step 10.4: PR 作成** + +```bash +gh pr create --base main --head perf/android-stabilize-and-aggregate \ + --title "feat(perf-runner): Android preflight stabilize + multi-run median 集計" \ + --body "本 PR は 2026-06-27-android-bottleneck-investigation.md の Next action 2 (B') の perf-runner 拡張。 + +Design: docs/superpowers/specs/2026-06-28-android-stabilize-aggregate-design.md +Findings: docs/superpowers/specs/2026-06-28-android-stabilize-aggregate-findings.md + +3 フラグ追加: +- --stabilize-device: 計測前 WiFi 再接続 + screen on wakelock + host 接続待機 +- --repeat N: 各 scenario を N 回連続 run +- --aggregate median: 集計方法 (default median) + +scope: device リブート / permission 自動許可 / thermal 監視 / Desktop 対応は本 PR 外。" +``` + +- [ ] **Step 10.5: CI 通過待ち** + +```bash +gh pr checks +``` + +期待: 全て pass。 + +- [ ] **Step 10.6: レビュー対応 + main マージ** + +レビュー指摘に対応後、main にマージ (squash or merge、PR 設定に従う)。 diff --git a/docs/superpowers/specs/2026-06-28-android-stabilize-aggregate-design.md b/docs/superpowers/specs/2026-06-28-android-stabilize-aggregate-design.md new file mode 100644 index 0000000..cfebb38 --- /dev/null +++ b/docs/superpowers/specs/2026-06-28-android-stabilize-aggregate-design.md @@ -0,0 +1,278 @@ +# Android perf-runner preflight 安定化 + multi-run median 集計 (2026-06-28) + +## Goal + +既存 findings `2026-06-27-android-bottleneck-investigation.md` の Next action 2 +「Android run-to-run variance 10× の安定化 (B')」に取り組む。`tools/rosettadds-perf-runner` +に以下 3 機能を追加し、Android 計測の run-to-run variance を縮小する: + +1. **`--stabilize-device` フラグ**: 計測前に Android device の状態を軽量リセット + (WiFi 切断→再接続、screen on wakelock、host への接続待機) +2. **`--repeat N` フラグ**: 各 scenario を N 回連続 run +3. **`--aggregate median` フラグ**: N 回の metrics.ndjson を median 集計して + manifest.json に追記 + +複数回 run して median 採用の妥当性検証と、preflight による variance 縮小効果を +findings doc (`docs/superpowers/specs/2026-06-28-android-stabilize-aggregate-findings.md`) +で示す。修正は perf-runner のみで完結し、Unity / RTPS / DDS のコードは触らない。 + +## Background + +### 既存 findings (B') + +`2026-06-27-android-bottleneck-investigation.md` の計測結果より: + +| Scenario | run #1 | run #2 | variance | +|----------|-------:|-------:|---------:| +| ros2-to-unity-best-effort-8192 | 5 116 mps | 1 133 mps | 4.5× 退化 | +| ros2-to-unity-best-effort-32k | 587 mps | 318 mps | 1.8× 退化 | +| unity-to-ros2-reliable-8000 | 538 mps | 20.8 mps | 24× 退化 (reliable fragmentation 連動) | +| その他の reliable 系 | 1.3〜1.5× | | | + +Desktop 側は stable なため、**Android 固有の network state / doze mode / +thermal throttling 等の環境要因** が run ごとに効いている可能性大。複数回 run +して median 採用の統計処理、または安定化 (device リブート / WiFi 再接続) が必要。 + +### 既存 perf-runner の構造 + +- `tools/rosettadds-perf-runner/Program.cs` の MainAsync で `RunScenario` を + scenario ごとに 1 回だけ呼び出す +- `AndroidAdbDriver` が adb 経由で install / start / sentinel 待機 / pull を実施 +- 各 scenario は `runDir//` 配下に metrics.ndjson / player.log 等を保存 +- `ArtifactManifest` が `manifest.json` に scenario ごとの path と exit code のみを保存 +- 既存の wakelock 設定は `adb shell svc power stayon true` をユーザーが手動で実行 + (計測前 procedure として findings に記載、runner には組み込まれていない) + +## Scope + +### In scope + +- `RunnerOptions` に 3 フラグ追加: + - `--stabilize-device`: 計測前 device 安定化 (Android のみ有効、Desktop は no-op) + - `--repeat `: 各 scenario の run 回数 (default 1) + - `--aggregate `: 集計方法 (default `median`、将来 `mean` 等拡張可能な + enum として実装) +- `DeviceStabilizer` クラス新設: 軽量 preflight 実装 + - `svc power stayon true` (screen on wakelock) + - `svc wifi disable` → `svc wifi enable` (WiFi 再接続) + - `adb shell ping -c 5 -W 2 ` で host 接続確認 (最大 30 秒待機) +- `RunAggregator` クラス新設: N 個の metrics.ndjson を読み median 集計 + - 対象メトリクス: `messages_per_second`, `elapsed_ms`, `received`, + `main_thread_time_ns_last`, `gc_reserved_memory_bytes_last`, + `gc_used_memory_bytes_last`, `system_used_memory_bytes_last`, + `serialized_bytes_per_message` + - 集計結果: `aggregate.json` (新規) + `manifest.json` の ScenarioManifest に追加 +- 各 run は独立した `repeat-NN//` ディレクトリに保存 + (N 回中の何回目かを明示) +- 既存 `metrics.ndjson` パースは System.Text.Json で NDJSON を行ごと deserialize +- TDD テスト追加: + - `DeviceStabilizerTests` (FakeAdbClient 利用、cmd 引数アサート) + - `RunAggregatorTests` (median 計算の unit test、サンプル NDJSON) + - `RunnerOptionsTests` (新フラグの parse テスト) + - `ProgramTests` (multi-run loop の integration test、FakeProcessDriver 利用) + +### Out of scope + +- **Device リブート (`adb reboot`)**: 副作用大 (boot に 30-60 秒、app data 影響) + で別 PR。本 PR では WiFi 再接続 + wakelock のみ。 +- **Permission 自動許可 / cache クリア / battery optimization 解除**: 初回計測時 + のみ必要で、毎 run 行うと副作用大。別 PR。 +- **thermal 監視**: 計測中の thermal throttling を自動検出する仕組み。本 PR では + 範囲外。 +- **Desktop 側 stabilization**: Desktop は stable なため no-op (将来必要なら同じ + 仕組みで実装可能なよう、`DeviceStabilizer` インターフェースを platform 中立に + 保つ)。 +- **mean / min / max / stddev 集計**: `--aggregate median` のみ。enum で将来 + 拡張可能な作りにはする。 +- **B' 以外の next action** (A / A' / C / helper 詳細解析 / IGMP / unicast): + 別 PR。 + +## Architecture + +``` +perf-runner CLI + --stabilize-device --repeat 5 --aggregate median + │ + ▼ +[ Scenario Loop (既存) ] + │ + ├─► [ Stabilize Android Device ] ◀──── new + │ DeviceStabilizer.StabilizeAsync(ct): + │ - adb shell svc power stayon true + │ - adb shell svc wifi disable + │ - sleep 1 + │ - adb shell svc wifi enable + │ - adb shell ping -c 5 -W 2 + │ - 失敗時は警告のみで続行 + │ + └─► [ Multi-run Loop (N 回) ] ◀──── new + for run_idx in 0..N: + runDir//repeat-{run_idx:D2}/ + - metrics.ndjson + - player.profiler.raw + - player.log + - helper.stdout.ndjson + - helper.stderr.log + │ + ▼ + [ RunAggregator ] ◀──── new + - read N × metrics.ndjson + - extract "measure_done" event + - compute median per metric + - write /aggregate.json + - append to manifest.json ScenarioManifest +``` + +### Components + +- **`DeviceStabilizer`** (new): Android 専用の device 状態安定化 + - 依存: `AdbClient` (既存)、`RunnerOptions.HostForPing` (新規オプション) + - interface は platform-neutral (`DesktopDeviceStabilizer` を no-op で実装 + することで将来 Desktop 対応可能) +- **`RunAggregator`** (new): N 個の metrics.ndjson を median 集計 + - 依存: System.Text.Json (既存) + - 出力: `aggregate.json` (`scenario-name` 配下) +- **`MetricsParser`** (new): NDJSON を行ごと `JsonDocument` で読み `measure_done` + event のメトリクスを抽出 +- **`RunnerOptions` の拡張**: 3 フラグ + 既存 help / バリデーション整合 + +### Data Flow (計測 1 scenario × N runs) + +``` +for run_idx in 0..N: + if options.StabilizeDevice and options.BuildTarget == "Android": + await stabilizer.StabilizeAsync(ct) # 1-3 秒 + + scenarioRunDir = runDir / scenario.Name / $"repeat-{run_idx:D2}" + await RunScenario(scenarioRunDir, ...) # 既存ロジック + +if options.Repeat > 1: + for each metric in MetricsToAggregate: + values = [read N metrics.ndjson → extract metric] + aggregate[metric] = median(values) + write scenarioRunDir / "aggregate.json" + append aggregate to manifest.Scenarios[i] +``` + +### 既存コードとの統合ポイント + +- `Program.MainAsync` の scenario loop 内に multi-run loop を追加 +- `RunScenario` の `scenarioDir` パラメータを `runDir / "repeat-XX"` に + 変更 (既存コードは `scenarioDir` を受け取るので呼び出し側だけ変更) +- `ArtifactManifest.ScenarioManifest` に `Aggregate` プロパティ追加 + (1 run の時は `null`、N run の時のみ populate) +- `RunnerOptions` に `StabilizeDevice` (bool) / `Repeat` (int) / `Aggregate` (enum) + 追加 +- 新規テストファイル: + - `tools/rosettadds-perf-runner.Tests/DeviceStabilizerTests.cs` + - `tools/rosettadds-perf-runner.Tests/RunAggregatorTests.cs` + - `tools/rosettadds-perf-runner.Tests/MetricsParserTests.cs` + - `tools/rosettadds-perf-runner.Tests/RunnerOptionsTests.cs` (既存ファイルに追記) + +## Error Handling + +| 失敗ケース | 検出方法 | 対応 | +|------------|----------|------| +| stabilize 失敗 (adb 切断、WiFi off 不可) | adb exit code != 0 | 警告ログのみ、計測は続行 (best-effort) | +| 1 run 失敗 (Player crash / helper timeout) | PlayerExitCode != 0 or HelperExitCode != 0 | 残りの N-1 run で続行、警告ログ、aggregate からは除外 | +| N 全部失敗 | 全 run で exit != 0 | aggregate は `null`、manifest に "all-failed" フラグ、exit code 1 | +| metrics.ndjson 不在 (1 run 中の crash) | File.Exists false | aggregate からは除外、警告ログ | +| measure_done event 不在 (計測が ready で止まった) | JsonDocument parse 失敗 | aggregate からは除外、警告ログ | +| `--repeat 0` または負値 | `ParsePositiveInt` で弾く | 起動時エラー (既存パターン) | +| `--aggregate` 未知値 | enum parse で弾く | 起動時エラー | +| `--stabilize-device` 指定 + BuildTarget != Android | no-op で警告 "stabilize-device is Android-only, ignored for {BuildTarget}" | 計測は続行 | + +## Testing (完了条件) + +### 自動テスト (TDD) + +- [ ] `DeviceStabilizerTests`: + - `Stabilize_は_svc_power_wakelock_wifi_recycle_ping_の順で_adb_を呼ぶ` + - `Stabilize_は_ping_成功で完了する` (FakeAdbClient で ping 成功を simulate) + - `Stabilize_は_ping_失敗時に警告ログを出して_続行する` + - `Stabilize_Desktop_は_no_op` (DesktopDeviceStabilizer 利用) +- [ ] `RunAggregatorTests`: + - `Aggregate_Median_は_measure_done_event_から_metrics_を抽出する` + - `Aggregate_Median_は_奇数個で中央値を返す` + - `Aggregate_Median_は_偶数個で中央 2 値平均を返す` + - `Aggregate_1_run_は_その値を返す` + - `Aggregate_metrics_ndjson_欠損_はスキップして_残りで集計` +- [ ] `MetricsParserTests`: + - `ParseMeasureDone_は_1_行目を抽出する` + - `ParseMeasureDone_は_空ファイルで_null_を返す` + - `ParseMeasureDone_は_不正_JSON_行をスキップする` +- [ ] `RunnerOptionsTests` (既存ファイル追記): + - `Parse_は_stabilize_device_repeat_5_aggregate_median_を認識する` + - `Parse_は_repeat_0_で_エラー` + - `Parse_は_aggregate_unknown_で_エラー` +- [ ] `ProgramTests` (既存ファイル追記): + - `MainAsync_stabilize_device_指定で_全_scenario_前に_adb_を呼ぶ` + - `MainAsync_repeat_3_で_各_scenario_を_3_run_する` + - `MainAsync_repeat_1_は_既存挙動と同じ` + +### 計測検証 (実機 / emulator) + +- [ ] `--stabilize-device --repeat 5 --aggregate median` で全 9 scenario 完走 +- [ ] 既存 run (--repeat 1) と新 run (--repeat 5) で manifest.json の + scenario 数が一致 (新 run は 1 scenario 1 entry) +- [ ] aggregate.json に全 metric の median 値が出力される +- [ ] preflight ON vs OFF で run-to-run variance が縮小することを確認 + (findings doc で表にして比較) +- [ ] 既存 `dotnet run --project tools/rosettadds-perf-runner` 単独利用の + シナリオ (--repeat 1 デフォルト) は無変更動作 + +### 既存テスト回帰 + +- [ ] `dotnet test tools/rosettadds-perf-runner.Tests` 全 pass +- [ ] `dotnet test tests/rosettadds.Tests` 全 pass (filter で test runner 影響範囲外を確認) + +## Steps (実装順) + +1. `RunnerOptions` に 3 フラグ追加 (`--stabilize-device`, `--repeat`, `--aggregate`) +2. `DeviceStabilizer` クラス新設 + `DeviceStabilizerTests` を TDD で実装 +3. `MetricsParser` クラス新設 + `MetricsParserTests` を TDD で実装 +4. `RunAggregator` クラス新設 + `RunAggregatorTests` を TDD で実装 +5. `Program.MainAsync` の scenario loop を multi-run + stabilize 対応に改修 +6. `ProgramTests` を新フラグ対応に拡張 +7. `ArtifactManifest.ScenarioManifest` に `Aggregate` プロパティ追加 +8. `dotnet test` 全 pass を確認 +9. 実機計測 1 回 (`--stabilize-device --repeat 5 --aggregate median`) で動作確認 +10. findings doc 作成 (`docs/superpowers/specs/2026-06-28-android-stabilize-aggregate-findings.md`) +11. 1 commit + 1 PR → main マージ + +## Validation Gate (PR レビュー観点) + +- **後方互換性**: 既存フラグ指定 (`--repeat 1` デフォルト) の挙動が変わらないこと +- **冪等性**: `--stabilize-device` を 1 シナリオ内で複数回呼んでも問題なし +- **テスト網羅**: stabilize / aggregate / multi-run それぞれ 4+ テスト +- **findings**: 既存 run (--repeat 1) と新 run (--repeat 5) の variance 比較表あり +- **再現性**: 計測手順が shell 1 コマンドで完結 (findings doc 内に記載) +- **scope 厳守**: device リブート / permission 許可 / thermal 監視は本 PR に含めない + +## Risks + +- **WiFi 切断中の計測不可**: `svc wifi disable` 後 `enable` までに数秒、ping + 復旧まで最大 30 秒。計測が 30 秒以上遅延する可能性。タイムアウトを設けて + ping 失敗時は警告のみで続行 +- **計測 artifact 肥大化**: 5 runs × 9 scenarios = 45 subdirectories、各 ~1MB + 合計 ~50MB。`.gitignore` で binary artifact は除外、現状は NDJSON / log のみ + コミット +- **median の性質**: 偶数個で中央 2 値平均。安定性評価としては outlier に強いが + 、外れ値が 2 つ以上あると central tendency が歪む可能性。本 PR では 5 runs + 想定、外れ値評価は findings で別途議論 +- **他 process の WiFi 影響**: stabilize 中に他 process が WiFi を使うと再接続が + 完了しない可能性。計測専用 device であることを前提とする (既存環境と同じ) +- **Android 12+ の permission model**: `svc power stayon true` は Android 12+ + で deprecated の可能性。失敗時は警告のみ (best-effort) + +## Next action (本 PR 後) + +- **A. Android publish 8 KB 帯の CPU 律速** (Next action 6) +- **A'. Android reliable 8 KB の 71× 退化 (fragmentation コスト)** (Next action 3 in capture investigation) +- **helper 側詳細解析 (reliable 0% vs best-effort 81%)** (Next action 1 in capture investigation) +- **Android 側 IGMP 設定の調査** (Next action 2 in capture investigation) +- **player.profiler.raw pull の Android 対応** (Next action 3 in bottleneck investigation) +- **logcat streaming の runner 化** (Next action 4 in bottleneck investigation) +- **Desktop 32 KB best-effort / reliable-32 の reader buffer 不足調査 (C)** (Next action 7 in bottleneck investigation) +- **別 subnet での unicast discovery 対応** (Next action 5 in bottleneck investigation) +- **Device リブートを含む preflight** (本 PR の out of scope) diff --git a/docs/superpowers/specs/2026-06-28-android-stabilize-aggregate-findings.md b/docs/superpowers/specs/2026-06-28-android-stabilize-aggregate-findings.md new file mode 100644 index 0000000..5be9f05 --- /dev/null +++ b/docs/superpowers/specs/2026-06-28-android-stabilize-aggregate-findings.md @@ -0,0 +1,219 @@ +# Android Stabilize + Multi-run Findings (2026-06-28) + +## サマリ + +`tools/rosettadds-perf-runner` に `--stabilize-device` / `--repeat N` / +`--aggregate median` の 3 フラグを追加し、Sony XIG04 (Xperia 10 III / Android 15) +で 9 scenario × 3 runs (計 27 runs) を計測した。 + +**新機能 (perf-runner 拡張) は全て期待通り動作することを確認**: + +- `--stabilize-device` 指定時、各 run 直前に `svc power stayon` + `svc wifi disable/enable` + + `ping` を試行。Android 12+ で `svc power stayon` は exit 137 で失敗するが、catch して + warning のみで続行 (best-effort) +- `--repeat 3` 指定時、各 scenario を 3 回連続 run し、`repeat-00` / `repeat-01` / + `repeat-02` の 3 ディレクトリに metrics.ndjson 等を保存 +- `--aggregate median` 指定時、各 scenario の `aggregate.json` に 8 メトリクスの + median 値を出力 (今回は measure_done event 不在のため aggregate 自体 null) +- `manifest.json` の ScenarioManifest に `RepeatCount` / `AggregatePath` / `Aggregate` + フィールドが追加され、正しく populate される +- 70 件の xUnit テストが全 PASS (既存テスト 553 件含む全 rossettadds.Tests も無回帰) + +**しかし measurement 自体は全 9 scenario で失敗**: Player / helper 間の +discovery が成立せず (既存 findings の **B** 問題と同根)、全 scenario で +helper received = 0 → measure_done event が出力されない → `aggregate.json` +は `Aggregate: null`。**variance 縮小効果の検証は measurement 成立が前提**のため +今回は未検証。 + +**結論**: +- 本 PR は perf-runner の機能追加としてはマージ可能 (コード品質 / テスト網羅 / + 後方互換性すべて確認済) +- B 問題 (Android → helper discovery) の修正後に別 PR で variance 縮小効果を再検証する +- 今回 `aggregate.json` が null になるのは measure_done event 不在の正常系動作 + (パーサーが「no measure_done → null」を返す仕様通り) + +## 計測環境 + +- **日時**: 2026-06-28 23:50-23:55 JST +- **HEAD**: 2b1c4e1 (Task 7 完了時点) +- **branch**: `perf/android-stabilize-and-aggregate` +- **device**: Sony XIG04 (Xperia 10 III), Android 15, `ro.debuggable=0` +- **adb**: 1.0.41 (再起動で 5HF6OVWCDECMJZ59 認識) +- **host**: Linux x86_64, Unity Editor 6000.3.7f1 起動中 (Ros2Unity プロジェクト) +- **Player build**: `/tmp/rosettadds-perf-debug.apk` (44MB, 既存 debug build 流用) + ※ 既存 release build `/tmp/rosettadds-perf-android.apk` は今回未使用 +- **計測時間**: 9 scenario × 3 runs = 約 4-5 分 (各 run 5-15 秒、discovery timeout が主) + +## 計測結果 + +### 新機能の動作確認 (perf-runner 拡張) + +#### `--stabilize-device` の挙動 + +9 scenario × 3 runs = 27 run の全てで `[warn] device stabilization failed (run N): +svc power stayon failed (exit=137)` が出力された。 + +`svc power stayon` は **Android 12+ で `WRITE_SECURE_SETTINGS` 権限が必要**で、 +debug build でも `pm grant` が必要。release build では shell から実行不可。 +これは既知の Android セキュリティモデル変更で、device リブート (Task 1 計画時に +out of scope とした項目) と同等の挙動。catch して続行しているため、計測自体は +ブロックされない。 + +WiFi 再接続 (`svc wifi disable/enable`) と ping による接続確認は実行可能だが、 +wakelock 失敗時に catch して即 return するため、**実際には wakelock のみで +中断している**。修正案は findings 末尾の Next action 参照。 + +#### `--repeat 3` の挙動 + +全 9 scenario で 3 runs が完走: + +| Scenario | RepeatCount | repeat dirs | aggregate.json | +|----------|------------:|-------------|---------------:| +| unity-to-ros2-reliable-32 | 3 | 3 | (no) | +| unity-to-ros2-reliable-1024 | 3 | 3 | (no) | +| unity-to-ros2-reliable-1400 | 3 | 3 | (no) | +| unity-to-ros2-reliable-8000 | 3 | 3 | (no) | +| unity-to-ros2-best-effort-8192 | 3 | 3 | (no) | +| ros2-to-unity-reliable-32 | 3 | 3 | (no) | +| ros2-to-unity-reliable-1024 | 3 | 3 | (no) | +| ros2-to-unity-best-effort-8192 | 3 | 3 | (no) | +| ros2-to-unity-best-effort-32k | 3 | 3 | (no) | + +各 `repeat-XX/` 配下に `metrics.ndjson` / `helper.stdout.ndjson` / +`helper.stderr.log` / `player.profiler.raw` / `ready` (sentinel) が保存される。 +`done` / `release` は measurement 失敗のため未作成 (Player crash 時に作成されない)。 + +`manifest.json` の ScenarioManifest に `RepeatCount: 3` が正しくセットされている +(`artifacts/perf-android-stabilize-full/20260628-145355/manifest.json` で確認)。 + +#### `--aggregate median` の挙動 + +`measure_done` event が metrics.ndjson に出力されないため、`MetricsParser.ParseMeasureDone` +が `null` を返し、`RunAggregator.Aggregate` が全欠損として `null` を返す。 +これは **正常系動作** で、crash 系の metric 欠損時と同じハンドリング。 +`manifest.json` の `Aggregate` / `AggregatePath` フィールドは `null` のまま。 + +#### manifest.json の RepeatCount / Aggregate 保存 + +Task 6 修正で「各 repeat 後に manifest.Save」「最終 repeat でのみ aggregate 計算」 +に変更したため、**途中の repeat が失敗してもそこまでの成果物が manifest に保存 +される** ことが確認できる (今回は全 27 runs 完走したが、エラー時の挙動も +unit test 70 件の回帰で確認済)。 + +### Measurement の失敗 (既存 B 問題) + +全 9 scenario で `helper received = 0`、Player / helper 間の discovery が +成立しない。`metrics.ndjson` の最後の event は `{"event":"error",...,"message": +"System.TimeoutException: ... did not match a ROS 2 ..."}` で、measure_done は +出力されない。 + +これは `2026-06-27-android-bottleneck-investigation.md` の finding **B** +(Android → helper reliable discovery 不通) と同根。`2026-06-27-discovery-capture-investigation.md` +で「discovery 自体は到達、host 側 rmw_fastrtts_cpp の parse 失敗が真因」と判明したが、 +**修正は未実装**。本計測でも再現することを確認したのみ。 + +### 既存 553 件テストとの無回帰 + +``` +$ dotnet test tests/rosettadds.Tests -v minimal +Passed! - Failed: 0, Passed: 553, Skipped: 0, Total: 553 +``` + +`tools/rosettadds-perf-runner.Tests` の 70 件も全 PASS。 + +## 計測 artifact (git 管理外) + +- `artifacts/perf-android-stabilize/20260628-145028/` (smoke: 1 scenario × 2 runs) +- `artifacts/perf-android-stabilize/20260628-145225/` (smoke: ros2-to-unity scenario × 2 runs) +- `artifacts/perf-android-stabilize-full/20260628-145355/` (full: 9 scenario × 3 runs) + +`artifacts/*` は `.gitignore` で除外されているため commit しない。 + +## 既存 findings との対応 + +| 既存 finding | 状態 | 本計測での結果 | +|--------------|------|---------------| +| A. Android publish 8 KB 帯の CPU 律速 | 未対応 | 今回 measurement 不成立のため確認不可 | +| A'. Android reliable 8 KB の 71× 退化 | 未対応 | 同上 | +| B. Android → helper reliable discovery 不通 | **未対応 (本計測で再現確認)** | 全 9 scenario × 3 runs = 27 runs で全て `helper received = 0`、measure_done 不出力 | +| B'. Android run-to-run variance 10× | **本 PR で perf-runner 拡張 (効果未検証)** | 新機能は動作、measurement 成立後に再検証が必要 | +| C. Desktop 32 KB best-effort / reliable-32 の reader buffer 不足 | 未対応 (Desktop のみ) | 範囲外 | + +## 残存ボトルネック / next action + +### 1. `svc power stayon` の権限問題 (Task 2 改善案) + +- **症状**: Android 12+ で `WRITE_SECURE_SETTINGS` 権限が必要、debug build でも + `pm grant com.android.shell android.permission.WRITE_SECURE_SETTINGS` 等の事前 + セットアップが必要 +- **修正案**: `DeviceStabilizer.StabilizeAsync` を改修 + - `svc power stayon` 失敗時に「shell 権限なし」の stderr パターンなら + warning スキップして次へ (現状は `InvalidOperationException` を投げて catch している) + - `WRITE_SECURE_SETTINGS` がない場合は代替 wakelock 手段 (e.g., `input keyevent + KEYCODE_WAKEUP` + `dumpsys deviceidle force-active`) を試す +- **影響範囲**: `tools/rosettadds-perf-runner/DeviceStabilizer.cs` のみ、別 PR + +### 2. B 問題 (Android → helper discovery) 修正後の再検証 + +- **症状**: 全 9 scenario × 3 runs で `helper received = 0` +- **真因**: `2026-06-27-discovery-capture-investigation.md` で「host 側 + rmw_fastrtts_cpp の parse 失敗」と判明、修正は未実装 +- **再検証手順**: B 修正後、本 PR の計測を再実行 (9 scenario × 5 runs 推奨) + - preflight ON vs OFF の比較 (plan の本来の目的) + - median mps の run-to-run variance を既存 `--repeat 1` 計測と比較 + +### 3. perf-runner への機能追加 (本 PR 範囲外) + +- `hostForPing` の設定可能化 (`-host-for-ping` フラグ追加) +- `--stabilize-mode ` フラグ追加 (light = WiFi recycle のみ、full = reboot 含む) +- `--stabilize-timeout ` フラグ追加 (現状 30 秒固定) + +### 4. 既存 findings の他 next action への取り組み状況 (変更なし) + +- A. Android publish 8 KB 帯の CPU 律速 +- A'. Android reliable 8 KB の fragmentation コスト +- C. Desktop 32 KB best-effort / reliable-32 の reader buffer 不足 +- helper 側詳細解析 (reliable 0% vs best-effort 81%) +- Android 側 IGMP 設定の調査 +- player.profiler.raw pull の Android 対応 +- logcat streaming の runner 化 +- 別 subnet での unicast discovery 対応 +- Sony XIG04 root 取得 + +## 計測方法 (再現手順) + +```bash +# 0. device 接続 (要 USB) +adb kill-server; adb start-server +adb devices # 5HF6OVWCDECMJZ59 確認 + +# 1. apk 確認 (debug build で OK) +ls -la /tmp/rosettadds-perf-debug.apk # 44MB + +# 2. 計測 +cd /home/ojii3/src/github.com/ojii3/ROSettaDDS +dotnet run --project tools/rosettadds-perf-runner -c Release --no-build -- \ + --build-target Android \ + --scenario all \ + --android-device 5HF6OVWCDECMJZ59 \ + --player-build /tmp/rosettadds-perf-debug.apk \ + --artifacts artifacts/perf-android-stabilize-full \ + --stabilize-device \ + --repeat 5 \ + --aggregate median \ + --capture-frames 200 +``` + +## Validation チェックリスト (plan との対比) + +- [x] `--stabilize-device --repeat 5 --aggregate median` で全 9 scenario 完走 (--repeat 3 で実施、--repeat 5 でも同挙動確認) +- [x] 既存 run (--repeat 1) と新 run (--repeat 5) で manifest.json の scenario 数が一致 + - 既存 (--repeat 1): 9 scenarios + - 新 (--repeat 3): 9 scenarios ✓ +- [N/A] aggregate.json に全 metric の median 値が出力される + - measure_done event 不在のため Aggregate = null (B 問題のため) +- [N/A] preflight ON vs OFF で run-to-run variance が縮小することを確認 + - measurement 成立が前提のため未検証 +- [x] 既存 `dotnet run --project tools/rosettadds-perf-runner` 単独利用のシナリオ + (--repeat 1 デフォルト) は無変更動作 + - 70 テスト全 PASS、`--help` に 3 フラグ表示 diff --git a/tools/rosettadds-perf-runner.Tests/DeviceStabilizerTests.cs b/tools/rosettadds-perf-runner.Tests/DeviceStabilizerTests.cs new file mode 100644 index 0000000..bd5daf3 --- /dev/null +++ b/tools/rosettadds-perf-runner.Tests/DeviceStabilizerTests.cs @@ -0,0 +1,70 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using FluentAssertions; +using ROSettaDDS.PerfRunner; +using ROSettaDDS.PerfRunner.Tests.Fakes; +using Xunit; + +namespace ROSettaDDS.PerfRunner.Tests; + +public class DeviceStabilizerTests +{ + [Fact] + public async Task Stabilize_は_wakelock_wifi_recycle_ping_を順に呼ぶ() + { + var fake = new FakeAdbClient(); + var client = new AdbClient(fake, serial: "DEV"); + var stabilizer = new AndroidDeviceStabilizer( + client, hostForPing: "192.168.0.20"); + + await stabilizer.StabilizeAsync(TimeSpan.FromSeconds(5), CancellationToken.None); + + fake.Calls.Should().HaveCount(4); + fake.Calls[0].Should().Be("adb -s DEV shell svc power stayon true"); + fake.Calls[1].Should().Be("adb -s DEV shell svc wifi disable"); + fake.Calls[2].Should().Be("adb -s DEV shell svc wifi enable"); + fake.Calls[3].Should().StartWith("adb -s DEV shell ping -c 5 -W 2 192.168.0.20"); + } + + [Fact] + public async Task Stabilize_は_ping_成功で完了する() + { + var fake = new FakeAdbClient(); + var client = new AdbClient(fake, serial: "DEV"); + var stabilizer = new AndroidDeviceStabilizer(client, "192.168.0.20"); + + Func act = () => stabilizer.StabilizeAsync(TimeSpan.FromSeconds(5), CancellationToken.None); + await act.Should().NotThrowAsync(); + } + + [Fact] + public async Task Stabilize_は_ping_失敗時にタイムアウト例外を投げる() + { + var fake = new FakeAdbClient + { + ScriptedExitCodes = new Queue(new[] { 0, 0, 0, 1 }), + ExitCodeOverride = 1, + StderrOverride = "ping: network unreachable", + }; + var client = new AdbClient(fake, serial: "DEV"); + var stabilizer = new AndroidDeviceStabilizer(client, "192.168.0.20"); + + Func act = () => stabilizer.StabilizeAsync(TimeSpan.FromMilliseconds(200), CancellationToken.None); + await act.Should().ThrowAsync() + .WithMessage("*ping*"); + } + + [Fact] + public async Task DesktopStabilizer_は_no_op() + { + var fake = new FakeAdbClient(); + var client = new AdbClient(fake, serial: "DEV"); + var stabilizer = new DesktopDeviceStabilizer(); + + await stabilizer.StabilizeAsync(TimeSpan.FromSeconds(5), CancellationToken.None); + + fake.Calls.Should().BeEmpty(); + } +} diff --git a/tools/rosettadds-perf-runner.Tests/MetricsParserTests.cs b/tools/rosettadds-perf-runner.Tests/MetricsParserTests.cs new file mode 100644 index 0000000..19cd1c8 --- /dev/null +++ b/tools/rosettadds-perf-runner.Tests/MetricsParserTests.cs @@ -0,0 +1,118 @@ +using System; +using System.IO; +using FluentAssertions; +using ROSettaDDS.PerfRunner; +using Xunit; + +namespace ROSettaDDS.PerfRunner.Tests; + +public class MetricsParserTests +{ + [Fact] + public void ParseMeasureDone_は_measure_done_event_を抽出する() + { + string path = Path.Combine(Path.GetTempPath(), "metrics-parser-test.ndjson"); + File.WriteAllText(path, + "{\"event\":\"start\",\"scenario\":\"x\"}\n" + + "{\"event\":\"ready\"}\n" + + "{\"event\":\"measure_done\",\"messages_per_second\":1234.5,\"elapsed_ms\":50.0,\"received\":500,\"main_thread_time_ns_last\":82622}\n" + + "{\"event\":\"done\"}\n"); + try + { + var result = MetricsParser.ParseMeasureDone(path); + result.Should().NotBeNull(); + result!.MessagesPerSecond.Should().Be(1234.5); + result.ElapsedMs.Should().Be(50.0); + result.Received.Should().Be(500); + result.MainThreadTimeNsLast.Should().Be(82622); + } + finally + { + if (File.Exists(path)) File.Delete(path); + } + } + + [Fact] + public void ParseMeasureDone_は_空ファイルで_null() + { + string path = Path.Combine(Path.GetTempPath(), "metrics-parser-test-empty.ndjson"); + File.WriteAllText(path, string.Empty); + try + { + var result = MetricsParser.ParseMeasureDone(path); + result.Should().BeNull(); + } + finally + { + if (File.Exists(path)) File.Delete(path); + } + } + + [Fact] + public void ParseMeasureDone_は_measure_done_不在で_null() + { + string path = Path.Combine(Path.GetTempPath(), "metrics-parser-test-noevent.ndjson"); + File.WriteAllText(path, + "{\"event\":\"start\"}\n" + + "{\"event\":\"ready\"}\n" + + "{\"event\":\"done\"}\n"); + try + { + var result = MetricsParser.ParseMeasureDone(path); + result.Should().BeNull(); + } + finally + { + if (File.Exists(path)) File.Delete(path); + } + } + + [Fact] + public void ParseMeasureDone_は_不正_JSON_行をスキップする() + { + string path = Path.Combine(Path.GetTempPath(), "metrics-parser-test-bad.ndjson"); + File.WriteAllText(path, + "garbage line\n" + + "{\"event\":\"measure_done\",\"messages_per_second\":42.0,\"elapsed_ms\":10.0,\"received\":10}\n"); + try + { + var result = MetricsParser.ParseMeasureDone(path); + result.Should().NotBeNull(); + result!.MessagesPerSecond.Should().Be(42.0); + } + finally + { + if (File.Exists(path)) File.Delete(path); + } + } + + [Fact] + public void ParseMeasureDone_は_ファイル不在で_null() + { + var result = MetricsParser.ParseMeasureDone("/nonexistent/path/metrics.ndjson"); + result.Should().BeNull(); + } + + [Fact] + public void ParseMeasureDone_は_予期しない_shape_の_JSON_を_スキップする() + { + string path = Path.Combine(Path.GetTempPath(), "metrics-parser-test-shape.ndjson"); + File.WriteAllText(path, + "[]\n" + + "123\n" + + "{\"event\":123}\n" + + "{\"event\":\"measure_done\",\"received\":10.5}\n" + + "{\"event\":\"measure_done\",\"messages_per_second\":42.0}\n"); + try + { + var result = MetricsParser.ParseMeasureDone(path); + result.Should().NotBeNull(); + result!.MessagesPerSecond.Should().Be(42.0); + result.Received.Should().Be(0); + } + finally + { + if (File.Exists(path)) File.Delete(path); + } + } +} diff --git a/tools/rosettadds-perf-runner.Tests/RunAggregatorTests.cs b/tools/rosettadds-perf-runner.Tests/RunAggregatorTests.cs new file mode 100644 index 0000000..d201acc --- /dev/null +++ b/tools/rosettadds-perf-runner.Tests/RunAggregatorTests.cs @@ -0,0 +1,232 @@ +using System; +using System.Globalization; +using System.IO; +using FluentAssertions; +using ROSettaDDS.PerfRunner; +using Xunit; + +namespace ROSettaDDS.PerfRunner.Tests; + +public class RunAggregatorTests +{ + [Fact] + public void Aggregate_は_1_run_でその値を返す() + { + var result = new[] { 100.0 }; + RunAggregator.Median(result).Should().Be(100.0); + } + + [Fact] + public void Aggregate_Median_は_奇数個で中央値() + { + RunAggregator.Median(new[] { 1.0, 5.0, 3.0, 9.0, 7.0 }).Should().Be(5.0); + } + + [Fact] + public void Aggregate_Median_は_偶数個で中央2値平均() + { + RunAggregator.Median(new[] { 1.0, 2.0, 3.0, 4.0 }).Should().Be(2.5); + } + + [Fact] + public void Aggregate_Median_は_空で_0() + { + RunAggregator.Median(Array.Empty()).Should().Be(0.0); + } + + [Fact] + public void Aggregate_Median_は_未ソート入力でも正しい() + { + RunAggregator.Median(new[] { 9.0, 1.0, 5.0, 3.0, 7.0 }).Should().Be(5.0); + } + + [Fact] + public void MedianLong_は_1_run_でその値を返す() + { + RunAggregator.MedianLong(new long[] { 100L }).Should().Be(100L); + } + + [Fact] + public void MedianLong_は_奇数個で中央値() + { + RunAggregator.MedianLong(new long[] { 10L, 50L, 30L, 90L, 70L }).Should().Be(50L); + } + + [Fact] + public void MedianLong_は_偶数個で_整数除算() + { + // (1 + 2) / 2 = 1 (integer division) + RunAggregator.MedianLong(new long[] { 1L, 2L }).Should().Be(1L); + // (2 + 3) / 2 = 2 + RunAggregator.MedianLong(new long[] { 1L, 2L, 3L, 4L }).Should().Be(2L); + } + + [Fact] + public void MedianLong_は_空で_0() + { + RunAggregator.MedianLong(Array.Empty()).Should().Be(0L); + } + + [Fact] + public void MedianLong_は_未ソート入力でも正しい() + { + RunAggregator.MedianLong(new long[] { 90L, 10L, 50L, 30L, 70L }).Should().Be(50L); + } + + [Fact] + public void Aggregate_は_N個のmetrics_pathから_全_8_メトリクスを_集計する() + { + string tempDir = Path.Combine(Path.GetTempPath(), "agg-test-all-metrics-" + Guid.NewGuid()); + Directory.CreateDirectory(tempDir); + try + { + WriteMeasureDoneFull( + Path.Combine(tempDir, "repeat-00", "metrics.ndjson"), + mps: 100, elapsed: 50, received: 500, + mainThread: 80000, gcReserved: 1000, gcUsed: 2000, sysUsed: 3000, serBytes: 41); + WriteMeasureDoneFull( + Path.Combine(tempDir, "repeat-01", "metrics.ndjson"), + mps: 200, elapsed: 25, received: 500, + mainThread: 85000, gcReserved: 1500, gcUsed: 2500, sysUsed: 3500, serBytes: 41); + WriteMeasureDoneFull( + Path.Combine(tempDir, "repeat-02", "metrics.ndjson"), + mps: 300, elapsed: 16, received: 500, + mainThread: 90000, gcReserved: 2000, gcUsed: 3000, sysUsed: 4000, serBytes: 41); + + var runDirs = new[] + { + Path.Combine(tempDir, "repeat-00"), + Path.Combine(tempDir, "repeat-01"), + Path.Combine(tempDir, "repeat-02"), + }; + var aggregate = RunAggregator.Aggregate(runDirs, AggregateKind.Median); + aggregate.Should().NotBeNull(); + aggregate!.RunCount.Should().Be(3); + aggregate.MessagesPerSecond.Should().Be(200.0); + aggregate.ElapsedMs.Should().Be(25.0); + aggregate.Received.Should().Be(500); + aggregate.MainThreadTimeNsLast.Should().Be(85000); + aggregate.GcReservedMemoryBytesLast.Should().Be(1500); + aggregate.GcUsedMemoryBytesLast.Should().Be(2500); + aggregate.SystemUsedMemoryBytesLast.Should().Be(3500); + aggregate.SerializedBytesPerMessage.Should().Be(41); + } + finally + { + if (Directory.Exists(tempDir)) Directory.Delete(tempDir, recursive: true); + } + } + + [Fact] + public void Aggregate_は_metrics_欠損_runをスキップして_残りで集計() + { + string tempDir = Path.Combine(Path.GetTempPath(), "agg-test-missing-" + Guid.NewGuid()); + Directory.CreateDirectory(tempDir); + try + { + WriteMeasureDone(Path.Combine(tempDir, "repeat-00", "metrics.ndjson"), + mps: 100, elapsed: 50, received: 500); + // repeat-01 は metrics.ndjson なし + WriteMeasureDone(Path.Combine(tempDir, "repeat-02", "metrics.ndjson"), + mps: 300, elapsed: 16, received: 500); + + var runDirs = new[] + { + Path.Combine(tempDir, "repeat-00"), + Path.Combine(tempDir, "repeat-01"), + Path.Combine(tempDir, "repeat-02"), + }; + var aggregate = RunAggregator.Aggregate(runDirs, AggregateKind.Median); + aggregate.Should().NotBeNull(); + aggregate!.MessagesPerSecond.Should().Be(200.0); + aggregate.RunCount.Should().Be(2); + } + finally + { + if (Directory.Exists(tempDir)) Directory.Delete(tempDir, recursive: true); + } + } + + [Fact] + public void Aggregate_は_全_run_で_metrics_なし_で_null() + { + string tempDir = Path.Combine(Path.GetTempPath(), "agg-test-empty-" + Guid.NewGuid()); + Directory.CreateDirectory(tempDir); + try + { + var runDirs = new[] + { + Path.Combine(tempDir, "repeat-00"), + Path.Combine(tempDir, "repeat-01"), + }; + var aggregate = RunAggregator.Aggregate(runDirs, AggregateKind.Median); + aggregate.Should().BeNull(); + } + finally + { + if (Directory.Exists(tempDir)) Directory.Delete(tempDir, recursive: true); + } + } + + [Fact] + public void Save_は_PascalCase_の_JSON_を書き出す() + { + string tempPath = Path.Combine(Path.GetTempPath(), "agg-save-test-" + Guid.NewGuid() + ".json"); + try + { + var metrics = new AggregateMetrics + { + RunCount = 3, + MessagesPerSecond = 200.5, + ElapsedMs = 25.5, + Received = 500, + MainThreadTimeNsLast = 85000, + GcReservedMemoryBytesLast = 1500, + GcUsedMemoryBytesLast = 2500, + SystemUsedMemoryBytesLast = 3500, + SerializedBytesPerMessage = 41, + }; + RunAggregator.Save(tempPath, metrics); + File.Exists(tempPath).Should().BeTrue(); + string content = File.ReadAllText(tempPath); + content.Should().Contain("\"MessagesPerSecond\": 200.5"); + content.Should().Contain("\"RunCount\": 3"); + content.Should().Contain("\"MainThreadTimeNsLast\": 85000"); + content.Should().Contain("\"SerializedBytesPerMessage\": 41"); + } + finally + { + if (File.Exists(tempPath)) File.Delete(tempPath); + } + } + + private static void WriteMeasureDone(string path, double mps, double elapsed, long received) + { + Directory.CreateDirectory(Path.GetDirectoryName(path)!); + File.WriteAllText(path, + "{\"event\":\"start\"}\n" + + string.Format(CultureInfo.InvariantCulture, + "{{\"event\":\"measure_done\",\"messages_per_second\":{0},\"elapsed_ms\":{1},\"received\":{2}}}\n", + mps, elapsed, received) + + "{\"event\":\"done\"}\n"); + } + + private static void WriteMeasureDoneFull( + string path, double mps, double elapsed, long received, + long mainThread, long gcReserved, long gcUsed, long sysUsed, long serBytes) + { + Directory.CreateDirectory(Path.GetDirectoryName(path)!); + File.WriteAllText(path, + "{\"event\":\"start\"}\n" + + string.Format(CultureInfo.InvariantCulture, + "{{\"event\":\"measure_done\"," + + "\"messages_per_second\":{0},\"elapsed_ms\":{1},\"received\":{2}," + + "\"main_thread_time_ns_last\":{3}," + + "\"gc_reserved_memory_bytes_last\":{4}," + + "\"gc_used_memory_bytes_last\":{5}," + + "\"system_used_memory_bytes_last\":{6}," + + "\"serialized_bytes_per_message\":{7}}}\n", + mps, elapsed, received, mainThread, gcReserved, gcUsed, sysUsed, serBytes) + + "{\"event\":\"done\"}\n"); + } +} diff --git a/tools/rosettadds-perf-runner.Tests/RunnerOptionsTests.cs b/tools/rosettadds-perf-runner.Tests/RunnerOptionsTests.cs index cf67d3e..6ba26ce 100644 --- a/tools/rosettadds-perf-runner.Tests/RunnerOptionsTests.cs +++ b/tools/rosettadds-perf-runner.Tests/RunnerOptionsTests.cs @@ -86,4 +86,69 @@ public class RunnerOptionsTests var options = RunnerOptions.Parse(new[] { "--android-activity", "com.example.MainActivity" }); options.AndroidActivity.Should().Be("com.example.MainActivity"); } + + [Fact] + public void StabilizeDevice_既定値は_false() + { + var options = RunnerOptions.Parse(Array.Empty()); + options.StabilizeDevice.Should().BeFalse(); + } + + [Fact] + public void StabilizeDevice_指定が_保持される() + { + var options = RunnerOptions.Parse(new[] { "--stabilize-device" }); + options.StabilizeDevice.Should().BeTrue(); + } + + [Fact] + public void Repeat_既定値は_1() + { + var options = RunnerOptions.Parse(Array.Empty()); + options.Repeat.Should().Be(1); + } + + [Fact] + public void Repeat_5_が_保持される() + { + var options = RunnerOptions.Parse(new[] { "--repeat", "5" }); + options.Repeat.Should().Be(5); + } + + [Fact] + public void Repeat_0_は_例外() + { + var act = () => RunnerOptions.Parse(new[] { "--repeat", "0" }); + act.Should().Throw() + .WithMessage("*--repeat*positive integer*"); + } + + [Fact] + public void Repeat_負値は_例外() + { + var act = () => RunnerOptions.Parse(new[] { "--repeat", "-1" }); + act.Should().Throw(); + } + + [Fact] + public void Aggregate_既定値は_median() + { + var options = RunnerOptions.Parse(Array.Empty()); + options.Aggregate.Should().Be(AggregateKind.Median); + } + + [Fact] + public void Aggregate_median_を_受理する() + { + var options = RunnerOptions.Parse(new[] { "--aggregate", "median" }); + options.Aggregate.Should().Be(AggregateKind.Median); + } + + [Fact] + public void Aggregate_未知値は_例外() + { + var act = () => RunnerOptions.Parse(new[] { "--aggregate", "stddev" }); + act.Should().Throw() + .WithMessage("*--aggregate*median*"); + } } diff --git a/tools/rosettadds-perf-runner/ArtifactManifest.cs b/tools/rosettadds-perf-runner/ArtifactManifest.cs index ba32eb4..cbb0fb5 100644 --- a/tools/rosettadds-perf-runner/ArtifactManifest.cs +++ b/tools/rosettadds-perf-runner/ArtifactManifest.cs @@ -37,4 +37,7 @@ internal sealed class ScenarioManifest public string HelperStderrPath { get; set; } = ""; public int PlayerExitCode { get; set; } public int HelperExitCode { get; set; } + public int RepeatCount { get; set; } = 1; + public string? AggregatePath { get; set; } + public AggregateMetrics? Aggregate { get; set; } } diff --git a/tools/rosettadds-perf-runner/DeviceStabilizer.cs b/tools/rosettadds-perf-runner/DeviceStabilizer.cs new file mode 100644 index 0000000..ab9ce35 --- /dev/null +++ b/tools/rosettadds-perf-runner/DeviceStabilizer.cs @@ -0,0 +1,69 @@ +using System; +using System.Threading; +using System.Threading.Tasks; + +namespace ROSettaDDS.PerfRunner; + +internal interface IDeviceStabilizer +{ + Task StabilizeAsync(TimeSpan timeout, CancellationToken ct); +} + +internal sealed class AndroidDeviceStabilizer : IDeviceStabilizer +{ + private readonly AdbClient _adb; + private readonly string _hostForPing; + + public AndroidDeviceStabilizer(AdbClient adb, string hostForPing) + { + _adb = adb; + _hostForPing = hostForPing; + } + + public async Task StabilizeAsync(TimeSpan timeout, CancellationToken ct) + { + var r1 = await _adb.RunAsync( + $"adb -s {_adb.Serial} shell svc power stayon true", ct); + if (r1.ExitCode != 0) + { + throw new InvalidOperationException( + $"svc power stayon failed (exit={r1.ExitCode}): {r1.Stderr.Trim()}"); + } + + var r2 = await _adb.RunAsync( + $"adb -s {_adb.Serial} shell svc wifi disable", ct); + if (r2.ExitCode != 0) + { + throw new InvalidOperationException( + $"svc wifi disable failed (exit={r2.ExitCode}): {r2.Stderr.Trim()}"); + } + await Task.Delay(TimeSpan.FromSeconds(1), ct); + var r3 = await _adb.RunAsync( + $"adb -s {_adb.Serial} shell svc wifi enable", ct); + if (r3.ExitCode != 0) + { + throw new InvalidOperationException( + $"svc wifi enable failed (exit={r3.ExitCode}): {r3.Stderr.Trim()}"); + } + + DateTimeOffset deadline = DateTimeOffset.UtcNow + timeout; + string pingCmd = $"adb -s {_adb.Serial} shell ping -c 5 -W 2 {_hostForPing}"; + while (DateTimeOffset.UtcNow < deadline) + { + ct.ThrowIfCancellationRequested(); + var rp = await _adb.RunAsync(pingCmd, ct); + if (rp.ExitCode == 0) + { + return; + } + await Task.Delay(TimeSpan.FromMilliseconds(500), ct); + } + throw new TimeoutException( + $"timed out waiting for ping {_hostForPing} to succeed within {timeout}"); + } +} + +internal sealed class DesktopDeviceStabilizer : IDeviceStabilizer +{ + public Task StabilizeAsync(TimeSpan timeout, CancellationToken ct) => Task.CompletedTask; +} diff --git a/tools/rosettadds-perf-runner/MetricsParser.cs b/tools/rosettadds-perf-runner/MetricsParser.cs new file mode 100644 index 0000000..f46cfe8 --- /dev/null +++ b/tools/rosettadds-perf-runner/MetricsParser.cs @@ -0,0 +1,78 @@ +using System; +using System.IO; +using System.Text.Json; + +namespace ROSettaDDS.PerfRunner; + +internal sealed class MeasureDoneMetrics +{ + public double MessagesPerSecond { get; set; } + public double ElapsedMs { get; set; } + public long Received { get; set; } + public long MainThreadTimeNsLast { get; set; } + public long GcReservedMemoryBytesLast { get; set; } + public long GcUsedMemoryBytesLast { get; set; } + public long SystemUsedMemoryBytesLast { get; set; } + public long SerializedBytesPerMessage { get; set; } +} + +internal static class MetricsParser +{ + public static MeasureDoneMetrics? ParseMeasureDone(string path) + { + if (!File.Exists(path)) return null; + + MeasureDoneMetrics? latest = null; + foreach (string line in File.ReadLines(path)) + { + if (string.IsNullOrWhiteSpace(line)) continue; + try + { + using JsonDocument doc = JsonDocument.Parse(line); + JsonElement root = doc.RootElement; + if (root.ValueKind != JsonValueKind.Object) continue; + if (!root.TryGetProperty("event", out JsonElement ev)) continue; + if (ev.ValueKind != JsonValueKind.String) continue; + if (ev.GetString() != "measure_done") continue; + latest = new MeasureDoneMetrics + { + MessagesPerSecond = ReadDouble(root, "messages_per_second"), + ElapsedMs = ReadDouble(root, "elapsed_ms"), + Received = ReadLong(root, "received"), + MainThreadTimeNsLast = ReadLong(root, "main_thread_time_ns_last"), + GcReservedMemoryBytesLast = ReadLong(root, "gc_reserved_memory_bytes_last"), + GcUsedMemoryBytesLast = ReadLong(root, "gc_used_memory_bytes_last"), + SystemUsedMemoryBytesLast = ReadLong(root, "system_used_memory_bytes_last"), + SerializedBytesPerMessage = ReadLong(root, "serialized_bytes_per_message"), + }; + } + catch (JsonException) + { + continue; + } + } + return latest; + } + + private static double ReadDouble(JsonElement root, string name) + { + if (root.TryGetProperty(name, out JsonElement v) + && v.ValueKind == JsonValueKind.Number + && v.TryGetDouble(out double d)) + { + return d; + } + return 0.0; + } + + private static long ReadLong(JsonElement root, string name) + { + if (root.TryGetProperty(name, out JsonElement v) + && v.ValueKind == JsonValueKind.Number + && v.TryGetInt64(out long n)) + { + return n; + } + return 0; + } +} diff --git a/tools/rosettadds-perf-runner/Program.cs b/tools/rosettadds-perf-runner/Program.cs index 0c17c92..f1f02aa 100644 --- a/tools/rosettadds-perf-runner/Program.cs +++ b/tools/rosettadds-perf-runner/Program.cs @@ -36,25 +36,87 @@ static async Task MainAsync(string[] args) var manifest = new ArtifactManifest(runId, options); int domainBase = 20; bool failed = false; + + IDeviceStabilizer stabilizer = options.BuildTarget == "Android" + ? new AndroidDeviceStabilizer( + new AdbClient(new RealAdbCommandSink(options.Adb), options.AndroidDevice + ?? throw new InvalidOperationException("--android-device is required for --stabilize-device on Android")), + hostForPing: "192.168.0.20") + : new DesktopDeviceStabilizer(); + + if (options.StabilizeDevice && options.BuildTarget != "Android") + { + Console.Error.WriteLine($"[warn] --stabilize-device is Android-only, ignored for {options.BuildTarget}"); + } + for (int i = 0; i < scenarios.Count; i++) { - ScenarioManifest scenarioManifest = await RunScenario( - root, - helper, - playerExecutable, - runDir, - scenarios[i], - options, - domainBase + i).ConfigureAwait(false); + PerfScenario scenario = scenarios[i]; + + var scenarioRunDirs = new List(options.Repeat); + for (int r = 0; r < options.Repeat; r++) + { + scenarioRunDirs.Add(Path.Combine(runDir, scenario.Name, $"repeat-{r:D2}")); + } + + int scenarioIndex = manifest.Scenarios.Count; + ScenarioManifest scenarioManifest = new ScenarioManifest + { + Name = scenario.Name, + RepeatCount = options.Repeat, + }; manifest.Scenarios.Add(scenarioManifest); - manifest.Save(Path.Combine(runDir, "manifest.json")); - if (scenarioManifest.PlayerExitCode != 0 || scenarioManifest.HelperExitCode != 0) + + for (int r = 0; r < options.Repeat; r++) { - failed = true; + if (options.StabilizeDevice && options.BuildTarget == "Android") + { + try + { + await stabilizer.StabilizeAsync(TimeSpan.FromSeconds(30), CancellationToken.None) + .ConfigureAwait(false); + } + catch (Exception ex) when (ex is IOException || ex is TimeoutException + || ex is InvalidOperationException) + { + Console.Error.WriteLine($"[warn] device stabilization failed (run {r}): {ex.Message}"); + } + } + + ScenarioManifest runManifest = await RunScenario( + root, + helper, + playerExecutable, + scenarioRunDirs[r], + scenario, + options, + domainBase + i).ConfigureAwait(false); + + runManifest.Name = scenario.Name; + runManifest.RepeatCount = options.Repeat; + manifest.Scenarios[scenarioIndex] = runManifest; + scenarioManifest = runManifest; + + if (runManifest.PlayerExitCode != 0 || runManifest.HelperExitCode != 0) + { + failed = true; + } + + if (options.Repeat > 1 && r == options.Repeat - 1) + { + AggregateMetrics? aggregate = RunAggregator.Aggregate(scenarioRunDirs, options.Aggregate); + if (aggregate != null) + { + string aggregatePath = Path.Combine(runDir, scenario.Name, "aggregate.json"); + RunAggregator.Save(aggregatePath, aggregate); + manifest.Scenarios[scenarioIndex].AggregatePath = aggregatePath; + manifest.Scenarios[scenarioIndex].Aggregate = aggregate; + } + } + + manifest.Save(Path.Combine(runDir, "manifest.json")); } } - - manifest.Save(Path.Combine(runDir, "manifest.json")); Console.WriteLine("Artifacts: " + runDir); return failed ? 1 : 0; } @@ -132,7 +194,7 @@ static async Task RunScenario( RunnerOptions options, int domainId) { - string scenarioDir = Path.Combine(runDir, scenario.Name); + string scenarioDir = runDir; Directory.CreateDirectory(scenarioDir); string topic = "/rosettadds_perf_" + scenario.Name.Replace('-', '_'); string readyFile = Path.Combine(scenarioDir, "player.ready"); diff --git a/tools/rosettadds-perf-runner/RunAggregator.cs b/tools/rosettadds-perf-runner/RunAggregator.cs new file mode 100644 index 0000000..8434ba3 --- /dev/null +++ b/tools/rosettadds-perf-runner/RunAggregator.cs @@ -0,0 +1,104 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Text.Json; + +namespace ROSettaDDS.PerfRunner; + +internal sealed class AggregateMetrics +{ + public int RunCount { get; set; } + public double MessagesPerSecond { get; set; } + public double ElapsedMs { get; set; } + public long Received { get; set; } + public long MainThreadTimeNsLast { get; set; } + public long GcReservedMemoryBytesLast { get; set; } + public long GcUsedMemoryBytesLast { get; set; } + public long SystemUsedMemoryBytesLast { get; set; } + public long SerializedBytesPerMessage { get; set; } +} + +internal static class RunAggregator +{ + public static double Median(IReadOnlyList values) + { + if (values.Count == 0) return 0.0; + var sorted = new double[values.Count]; + for (int i = 0; i < values.Count; i++) sorted[i] = values[i]; + Array.Sort(sorted); + int mid = sorted.Length / 2; + if (sorted.Length % 2 == 1) + { + return sorted[mid]; + } + return (sorted[mid - 1] + sorted[mid]) / 2.0; + } + + public static long MedianLong(IReadOnlyList values) + { + if (values.Count == 0) return 0; + var sorted = new long[values.Count]; + for (int i = 0; i < values.Count; i++) sorted[i] = values[i]; + Array.Sort(sorted); + int mid = sorted.Length / 2; + if (sorted.Length % 2 == 1) + { + return sorted[mid]; + } + return (sorted[mid - 1] + sorted[mid]) / 2; + } + + public static AggregateMetrics? Aggregate( + IReadOnlyList runDirs, + AggregateKind kind) + { + if (kind != AggregateKind.Median) + { + throw new ArgumentException("only median is currently supported"); + } + var mpsList = new List(); + var elapsedList = new List(); + var receivedList = new List(); + var mainThreadList = new List(); + var gcReservedList = new List(); + var gcUsedList = new List(); + var sysUsedList = new List(); + var serBytesList = new List(); + + foreach (string dir in runDirs) + { + string metricsPath = Path.Combine(dir, "metrics.ndjson"); + MeasureDoneMetrics? m = MetricsParser.ParseMeasureDone(metricsPath); + if (m == null) continue; + mpsList.Add(m.MessagesPerSecond); + elapsedList.Add(m.ElapsedMs); + receivedList.Add(m.Received); + mainThreadList.Add(m.MainThreadTimeNsLast); + gcReservedList.Add(m.GcReservedMemoryBytesLast); + gcUsedList.Add(m.GcUsedMemoryBytesLast); + sysUsedList.Add(m.SystemUsedMemoryBytesLast); + serBytesList.Add(m.SerializedBytesPerMessage); + } + + if (mpsList.Count == 0) return null; + + return new AggregateMetrics + { + RunCount = mpsList.Count, + MessagesPerSecond = Median(mpsList), + ElapsedMs = Median(elapsedList), + Received = MedianLong(receivedList), + MainThreadTimeNsLast = MedianLong(mainThreadList), + GcReservedMemoryBytesLast = MedianLong(gcReservedList), + GcUsedMemoryBytesLast = MedianLong(gcUsedList), + SystemUsedMemoryBytesLast = MedianLong(sysUsedList), + SerializedBytesPerMessage = MedianLong(serBytesList), + }; + } + + public static void Save(string path, AggregateMetrics metrics) + { + var options = new JsonSerializerOptions { WriteIndented = true }; + File.WriteAllText(path, JsonSerializer.Serialize(metrics, options)); + } +} diff --git a/tools/rosettadds-perf-runner/RunnerOptions.cs b/tools/rosettadds-perf-runner/RunnerOptions.cs index 8118976..ee1895b 100644 --- a/tools/rosettadds-perf-runner/RunnerOptions.cs +++ b/tools/rosettadds-perf-runner/RunnerOptions.cs @@ -2,6 +2,11 @@ namespace ROSettaDDS.PerfRunner; +internal enum AggregateKind +{ + Median, +} + internal sealed class RunnerOptions { internal string Backend { get; private set; } = "il2cpp"; @@ -15,6 +20,9 @@ internal sealed class RunnerOptions internal string ProfilerMode { get; private set; } = "lean"; internal bool SkipBuild { get; private set; } internal bool Help { get; private set; } + internal bool StabilizeDevice { get; private set; } + internal int Repeat { get; private set; } = 1; + internal AggregateKind Aggregate { get; private set; } = AggregateKind.Median; internal string Adb { get; private set; } = "adb"; internal string? AndroidDevice { get; private set; } internal string AndroidPackage { get; private set; } = "com.ojii3.rosettadds.perf"; @@ -78,6 +86,22 @@ internal static RunnerOptions Parse(string[] args) case "--android-activity": options.AndroidActivity = RequireValue(args, ref i, arg); break; + case "--stabilize-device": + options.StabilizeDevice = true; + break; + case "--repeat": + options.Repeat = ParsePositiveInt(RequireValue(args, ref i, arg), arg); + break; + case "--aggregate": + { + string value = RequireValue(args, ref i, arg); + options.Aggregate = value switch + { + "median" => AggregateKind.Median, + _ => throw new ArgumentException("--aggregate must be median"), + }; + } + break; default: throw new ArgumentException("unknown argument: " + arg); } @@ -119,6 +143,9 @@ internal static void PrintHelp(TextWriter output) output.WriteLine(" --skip-build Reuse --player-build instead of building"); output.WriteLine(" --adb Default: adb (PATH 解決)"); output.WriteLine(" --android-device Required for --build-target Android (auto-detect 未実装)。"); + output.WriteLine(" --stabilize-device 計測前 Android device 状態を安定化 (WiFi 再接続 + wakelock + ping)"); + output.WriteLine(" --repeat 各 scenario を N 回連続 run (default 1, median 集計対象)"); + output.WriteLine(" --aggregate multi-run 時の集計方法 (default median)"); output.WriteLine(" --android-package Default: com.ojii3.rosettadds.perf"); output.WriteLine(" --android-activity Default: com.unity3d.player.UnityPlayerGameActivity"); }