Parse io

Project.toml

@@ -9,9 +9,15 @@ CairoMakie = "13f3f980-e62b-5c42-98c6-ff1f3baf88f0"
 DelimitedFiles = "8bb1440f-4735-579b-a4ab-409b98df4dab"
 Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f"
 FFTW = "7a1cc6ca-52ef-59f5-83cd-3a7055c09341"
+GeometryBasics = "5c1252a2-5f33-56bf-86c9-59e7332b4326"
+Interpolations = "a98d9a8b-a2ab-59e6-89dd-64a1c18fca59"
+JSON3 = "0f8b85d8-7281-11e9-16c2-39a750bddbf1"
+LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
+Makie = "ee78f7c6-11fb-53f2-987a-cfe4a2b5a57a"
 Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7"
 ProgressMeter = "92933f4c-e287-5a05-a399-4b506db050ca"
 Roots = "f2b01f46-fcfa-551c-844a-d8ac1e96c665"
+Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
 StatsBase = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91"
 TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76"
 Unitful = "1986cc42-f94f-5a68-af5c-568840ba703d"
@@ -23,13 +29,19 @@ CairoMakie = ">=0.13"
 DelimitedFiles = "1"
 Distributions = ">=0.25"
 FFTW = "1.9.0"
+GeometryBasics = "0.5.10"
+Interpolations = "0.16.2"
+JSON3 = "1.14.3"
+LinearAlgebra = "1"
+Makie = "0.24.9"
 Printf = "1"
 ProgressMeter = "1"
 Roots = "2"
+Statistics = "1"
 StatsBase = ">=0.34"
 TOML = "1.0.3"
 Unitful = "1"
-WriteVTK = "1.21.2"
+WriteVTK = "1"
 julia = ">=1.10"
 
 [extras]

src/EntityTools.jl

@@ -3,6 +3,7 @@ module EntityTools
     using CairoMakie
     using Printf
     using StatsBase
+    using Statistics
     using Unitful
     using Distributions
     using StatsBase
@@ -11,13 +12,20 @@ module EntityTools
     using Printf
     using TOML
     using FFTW
-    using WriteVTK 
+    using WriteVTK
+    using Interpolations
+    using LinearAlgebra
+    using Makie
+    using GeometryBasics
+    using JSON3
 
     include("IO/adios/adios.jl")
     include("IO/adios/fields.jl")
     include("IO/adios/particles.jl")
     include("IO/adios/spectra.jl")
+    include("IO/adios/performance.jl")
     include("IO/vtk/write_vtk.jl")
+    include("IO/parse_output.jl")
     include("calc/phase.jl")
     include("calc/spectra.jl")
     include("calc/powerspectrum.jl")
@@ -29,10 +37,26 @@ module EntityTools
     include("units/time.jl")
     include("units/Bfield.jl")
     include("units/temperature.jl")
+    include("plotting/polar.jl")
 
     export phase_map,
             spectrum,
             parse_timing,
+            parse_output,
+            SimOutput,
+            StepData,
+            substep_series,
+            total_series,
+            active_particles,
+            timestep_durations,
+            elapsed_times,
+            remaining_times,
+            sim_times,
+            step_numbers,
+            species_series,
+            summary_table,
+            parse_compound_time,
+            adios2_throughput,
             EntityData,
             EntityUnits,
             find_closest_time,
@@ -42,5 +66,8 @@ module EntityTools
             bp_to_vtk,
             power_spectrum,
             get_Temp,
+            polar_fieldlines!,
+            polar_pcolor!,
+            polar_contour!
 
 end

src/IO/adios/performance.jl

@@ -0,0 +1,78 @@
+"""
+    adios2_throughput(path::AbstractString)
+
+Read an ADIOS2 BP5 `profiling.json` and return a NamedTuple with:
+
+- `aggregate_TBps`     : Σwbytes(all ranks) / max(durable_us) [decimal TB/s]
+- `per_rank_mean_GBps` : mean over writer ranks of wbytes / durable_us [decimal GB/s]
+- `per_rank_std_GBps`  : stddev of the same
+- `per_rank_raw_GBps`  : mean of wbytes / Σwrite.mus over writer ranks [decimal GB/s] —
+                         the raw POSIX write() rate, kept for reference; this measures
+                         the rate of memcpy into the page/client cache and overstates
+                         durable bandwidth, often above the filesystem spec
+- `n_writers`          : number of ranks that issued writes (BP5 aggregators)
+- `n_ranks`            : total ranks in the profile
+
+`durable_us` per rank is `ES + ES_close + DC_WaitOnAsync1 + DC_WaitOnAsync2`,
+i.e. the EndStep window plus the Close-time drain of BP5's async writer thread.
+With `AsyncWrite=ON`, `EndStep` returns before bytes are durable; the drain shows
+up as `DC_WaitOnAsync*` at Close. Using this window gives bandwidth bounded by
+what actually reached storage.
+
+Aggregate throughput uses `max(durable_us)` over all ranks (writers and
+non-writers) — non-writer ranks still block at Close waiting for the aggregator,
+and the wall clock is bounded by the slowest rank.
+"""
+function adios2_throughput(path::AbstractString)
+    raw = read(path, String)
+    # ADIOS2 sometimes emits a trailing comma before the closing array bracket
+    raw = replace(raw, r",(\s*\])" => s"\1")
+    data = JSON3.read(raw)
+
+    total_bytes          = 0
+    rank_eff_GBps        = Float64[]
+    rank_raw_GBps        = Float64[]
+    durable_times_us     = Float64[]
+
+    for rec in data
+        rank_bytes    = 0
+        rank_write_us = 0
+        for (k, v) in pairs(rec)
+            startswith(String(k), "transport_") || continue
+            v isa JSON3.Object || continue
+            rank_bytes    += get(v, :wbytes, 0)
+            wr             = get(v, :write, nothing)
+            wr === nothing || (rank_write_us += get(wr, :mus, 0))
+        end
+        total_bytes += rank_bytes
+
+        es      = Float64(get(rec, :ES_mus, 0))
+        esclose = Float64(get(rec, :ES_close_mus, 0))
+        wait1   = Float64(get(rec, :DC_WaitOnAsync1_mus, 0))
+        wait2   = Float64(get(rec, :DC_WaitOnAsync2_mus, 0))
+        durable_us = es + esclose + wait1 + wait2
+        durable_us > 0 && push!(durable_times_us, durable_us)
+
+        if rank_bytes > 0 && durable_us > 0
+            # bytes / μs == MB/s (decimal); /1000 -> GB/s
+            push!(rank_eff_GBps, rank_bytes / durable_us / 1000)
+        end
+        if rank_bytes > 0 && rank_write_us > 0
+            push!(rank_raw_GBps, rank_bytes / rank_write_us / 1000)
+        end
+    end
+
+    isempty(durable_times_us) && error("no ES/Close/Async timing entries found in $path")
+    wall_us = maximum(durable_times_us)
+    # bytes / μs == MB/s; /1e6 -> TB/s (decimal)
+    aggregate_TBps = total_bytes / wall_us / 1e6
+
+    return (
+        aggregate_TBps     = aggregate_TBps,
+        per_rank_mean_GBps = isempty(rank_eff_GBps) ? NaN : mean(rank_eff_GBps),
+        per_rank_std_GBps  = length(rank_eff_GBps) < 2 ? NaN : std(rank_eff_GBps),
+        per_rank_raw_GBps  = isempty(rank_raw_GBps) ? NaN : mean(rank_raw_GBps),
+        n_writers          = length(rank_eff_GBps),
+        n_ranks            = length(data),
+    )
+end