diff --git a/configure.ac b/configure.ac index 57b1dbebc..6e42ce82f 100644 --- a/configure.ac +++ b/configure.ac @@ -2191,7 +2191,7 @@ fi # HM case. AC_ARG_WITH([hm], [AS_HELP_STRING([--with-hm=], - [Human mobility scenario @<:@none, random-walk, river default: none@:>@] + [Human mobility scenario @<:@none, all, random-walk, river default: none@:>@] )], [with_hm="$withval"], [with_hm="none"] @@ -2201,18 +2201,17 @@ case "$with_hm" in none) AC_DEFINE([HM_CASE_NONE], [1], [No HM case]) ;; + "all") +# AC_DEFINE([HAVE_HM_ALL], 1, [Have all Human Mobility cases]) + AC_DEFINE([HM_CASE_RIVER], 1, [HM case River]) + AC_DEFINE([HM_CASE_RANDOMWALK], 1, [HM case Random Walk]) + ;; random-walk) AC_DEFINE([HM_CASE_RANDOMWALK], [1], [HM case Random Walk]) ;; river) AC_DEFINE([HM_CASE_RIVER], [1], [HM case River]) ;; - # river) - # AC_DEFINE([EXTERNAL_POTENTIAL_RIVER], [1], [River obstactle.]) - # ;; - # new-case) - # AC_DEFINE([HM_CASE_NEWSCENARIO], [1], [HM case New Scenario]) - # ;; *) AC_MSG_ERROR([Unknown HM case: $with_hm]) ;; diff --git a/examples/HumanMobility/doc/job_scheduling.md b/examples/HumanMobility/doc/job_scheduling.md new file mode 100644 index 000000000..f2cad1ef9 --- /dev/null +++ b/examples/HumanMobility/doc/job_scheduling.md @@ -0,0 +1,314 @@ +## SLURM job scheduling suite + +### Scripts + +Whenever you want to submit a job for a simulation (without or with profiling measurement) or for other applications (like visualisation), use the provided job submission script. + +* `./submit.sh --${type} -N ${nodes} -n ${ntasks} -c ${threads} [-p ${partition}]` .. a unified job submission script to submit a job of choice with possible types to select from: `gen` .. to generate data of humans and a map of rivers; `vis` .. to plot simulation results; `map` .. to plot only geography without humans; a set of other options to run a simulation (`run` .. without profiling; `likwid` .. profiled with _LIKWID_; `maqao` .. profiled with _MAQAO_; `aps` .. profiled with _Intel APS_; `vtune` .. profiled with _Intel VTune_; `advisor` .. profiled with _Intel Advisor_; `inspector` .. profiled with _Intel Inspector_; `scorep` .. profiled with _Score-P_) + +* `job.sh` .. a unified selector script that automatically detects partition and passes SLURM parameters to appropriate simulation scripts. Works on both COSMA5 and legacy COSMA partitions. + +* `gen.sh` .. a script to generate data of humans and a map of rivers (no parameters are passed) + +* `visualise.sh` .. a script to plot simulation results (no parameters are passed) + +* `map.sh` .. a script to plot only geography without humans (no parameters are passed) + +* `run.sh` .. a unified simulation script without profiling that automatically detects partition (COSMA5 vs COSMA) and execution mode (serial vs parallel) based on SLURM parameters + +* `run_perf.sh` .. a unified performance analysis script that handles all profiling tools (`--likwid`, `--maqao`, `--aps`, `--vtune`, `--advisor`, `--inspector`, `--scorep`) with automatic partition detection and resource-based directory naming + +#### Output Directory Structure + +The unified scripts automatically generate descriptive directory names based on: +- Tool used (for performance analysis) +- Number of nodes, MPI tasks, and threads +- Partition (COSMA5 vs COSMA) + +Examples: +- `data-rivers-1-1-64` (basic run on COSMA5: 1 node, 1 task, 64 threads) +- `data-rivers-1-1-16-cosma` (basic run on COSMA: 1 node, 1 task, 16 threads) +- `data-likwid-2-4-32` (LIKWID analysis on COSMA5: 2 nodes, 4 tasks, 32 threads each) +- `data-vtune-1-2-8-cosma` (VTune analysis on COSMA: 1 node, 2 tasks, 8 threads each) + +#### Usage examples + +Basic usage with defaults (1 node, 1 MPI rank, 64 threads on COSMA5): +```bash +./submit.sh --run # Basic simulation +./submit.sh --maqao # MAQAO performance analysis +./submit.sh --vtune # Intel VTune profiling +``` + +With custom resource allocation: +```bash +./submit.sh --likwid -N 1 -n 1 -c 32 # 1 node, 1 MPI rank, 32 threads +./submit.sh --advisor -N 1 -n 1 -c 16 # 1 node, 1 MPI rank, 16 threads +./submit.sh --inspector -N 1 -n 1 -c 8 # 1 node, 1 MPI rank, 8 threads +./submit.sh --scorep -N 1 -n 4 -c 8 # 1 node, 4 MPI ranks, 8 threads each +``` + +Partition-specific usage: +```bash +# COSMA5 (modern hardware, default) +./submit.sh --run -N 1 -n 1 -c 64 # Uses swift_intel2025 binary +./submit.sh --run -N 1 -n 1 -c 64 -p cosma5 # Explicit partition specification + +# COSMA (legacy hardware) +./submit.sh --run -N 1 -n 1 -c 16 -p cosma # Uses swift_cosma binary, max 16 cores/node +./submit.sh --vtune -N 2 -n 2 -c 8 -p cosma # Distributed across legacy nodes +``` + +#### Key Features of the Unified Approach + +* **Automatic partition detection**: Scripts automatically detect whether running on COSMA5 (modern) or COSMA (legacy) hardware and adapt accordingly +* **Dynamic directory naming**: Output directories are automatically named based on tool, resources, and partition (e.g., `data-likwid-1-2-32-cosma`) +* **Unified interface**: Same calling convention for all tools and partitions +* **Resource-aware configuration**: Automatic selection of appropriate SWIFT binaries and compiler versions based on partition + +**Benefits of Unified Scripts:** +- Single maintenance point for each functionality +- Automatic adaptation to different hardware +- Consistent naming scheme across all tools +- Reduced complexity and improved maintainability +- Future-proof design for new partitions (cosma7, cosma8, etc.) + +### Performance analysis + +For strong scaling, I begin with a minimal simulation of human mobility on a square 100 x 100 humans on 10 x 10 km (for minimum of either 1000 steps or 10 minutes). For weak scaling, I'd like to begin on 2 ranks from a square 100 x 100 humans on 10 x 10 km and then on 8 and 18 ranks, increasing the problem size accordingly. + +To conduct performance analysis, we have the following queues: + +* The _cosma5_ queue is comprised of the new COSMA5 nodes, a total of 3 nodes each with 256 cores and 1.5TB RAM +* The _cosma_ queue is comprised of the old COSMA nodes, a total of ~160 nodes each with 16 cores and 126GB RAM + +#### Strong scaling + +**Performance Analysis for Strong Scaling** + +For strong scaling analysis, we keep the problem size constant (100 x 100 humans on 10 x 10 km) and vary the number of computational resources: + +**Base Configuration (Serial Baseline):** +```bash +./submit.sh --run -N 1 -n 1 -c 1 # 1 node, 1 MPI rank, 1 thread (serial) +``` + +**Intra-node Scaling (COSMA5 queue - threading analysis):** +```bash +# Test threading efficiency within a single node +./submit.sh --run -N 1 -n 1 -c 16 # 1 node, 1 MPI rank, 16 threads +./submit.sh --run -N 1 -n 1 -c 32 # 1 node, 1 MPI rank, 32 threads +./submit.sh --run -N 1 -n 1 -c 64 # 1 node, 1 MPI rank, 64 threads +./submit.sh --run -N 1 -n 1 -c 128 # 1 node, 1 MPI rank, 128 threads +./submit.sh --run -N 1 -n 1 -c 256 # 1 node, 1 MPI rank, 256 threads (max COSMA5) +``` + +**Inter-node Scaling (COSMA5 queue - MPI analysis):** +```bash +# Test MPI scaling across multiple nodes with fixed threads per rank +./submit.sh --run -N 2 -n 2 -c 128 # 2 nodes, 2 MPI ranks, 128 threads each +./submit.sh --run -N 3 -n 3 -c 85 # 3 nodes, 3 MPI ranks, ~85 threads each (max COSMA5) +``` + +**COSMA queue comparison:** +```bash +# Compare with older COSMA hardware (can use many more nodes) +./submit.sh --run -N 1 -n 1 -c 16 -p cosma # 1 old COSMA node, 1 MPI rank, 16 threads (max) +./submit.sh --run -N 2 -n 2 -c 8 -p cosma # 2 old COSMA nodes, 2 MPI ranks, 8 threads each +./submit.sh --run -N 4 -n 4 -c 4 -p cosma # 4 old COSMA nodes, 4 MPI ranks, 4 threads each +./submit.sh --run -N 8 -n 8 -c 2 -p cosma # 8 old COSMA nodes, 8 MPI ranks, 2 threads each +``` + +**Performance Analysis Integration:** +```bash +# Run the same configurations with different profiling tools +./submit.sh --likwid -N 1 -n 1 -c 64 # LIKWID analysis on COSMA5 +./submit.sh --maqao -N 1 -n 1 -c 16 -p cosma # MAQAO analysis on COSMA +./submit.sh --vtune -N 2 -n 2 -c 128 # VTune analysis across 2 COSMA5 nodes +./submit.sh --vtune -N 3 -n 3 -c 85 # VTune analysis across 3 COSMA5 nodes (maximum) +``` + +#### Weak scaling + +**Performance Analysis for Weak Scaling** + +For weak scaling, we maintain constant work per computational unit by increasing problem size proportionally with the number of nodes. For 2D problems like human mobility, the problem size should scale with the square root of the number of computing nodes. + +**Scaling Strategy:** +- When we double the number of nodes, we should increase the linear dimension by ~1.414 (√2) +- For example: 100×100 humans → 141×141 humans → 173×173 humans (approximately) +- **Important**: Each scaling configuration requires its own input files with appropriately sized domains and human populations + +**Step 1: Generate Input Files for Each Configuration** + +Before running weak scaling tests, generate the appropriate input files for each node count: + +```bash +# For 2 nodes baseline (100x100 humans on 10x10 km) +./submit.sh --gen -- -n 100 -b 10000 +# Creates: humans-rivers-100.hdf5, river-rivers-100.hdf5 (or with -cosma suffix) + +# For 3 nodes (122x122 humans on 12.2x12.2 km) - COSMA5 maximum +./submit.sh --gen -- -n 122 -b 12200 +# Creates: humans-rivers-122.hdf5, river-rivers-122.hdf5 + +# For extended scaling on COSMA legacy hardware +# For 8 nodes (200x200 humans on 20x20 km) +./submit.sh --gen -- -n 200 -b 20000 +# Creates: humans-rivers-200.hdf5, river-rivers-200.hdf5 + +# For 18 nodes (300x300 humans on 30x30 km) +./submit.sh --gen -- -n 300 -b 30000 +# Creates: humans-rivers-300.hdf5, river-rivers-300.hdf5 +``` + +**Step 2: Run Weak Scaling Tests** + +Now run the simulations using the corresponding input files for each configuration: + +**Baseline (2 nodes, 100 x 100 humans on 10 x 10 km):** +```bash +./submit.sh --run -N 2 -n 2 -c 16 -p cosma -- --num-humans 100 # Uses humans-rivers-100.hdf5 +# or on COSMA5: +./submit.sh --run -N 2 -n 2 -c 128 -- --num-humans 100 # Uses humans-rivers-100.hdf5 +``` + +**Scale to 3 nodes (122 x 122 humans on 12.2 x 12.2 km) - COSMA5 maximum:** +```bash +./submit.sh --run -N 3 -n 3 -c 16 -p cosma -- --num-humans 122 # Uses humans-rivers-122.hdf5 +# or on COSMA5: +./submit.sh --run -N 3 -n 3 -c 64 -- --num-humans 122 # Uses humans-rivers-122.hdf5 +``` + +**Extended scaling on COSMA (legacy hardware with more nodes available):** +```bash +./submit.sh --run -N 8 -n 8 -c 16 -p cosma -- --num-humans 200 # Uses humans-rivers-200.hdf5 +./submit.sh --run -N 18 -n 18 -c 16 -p cosma -- --num-humans 300 # Uses humans-rivers-300.hdf5 +``` + +**Step 3: Profile at Scale** + +Run performance analysis with the appropriate input files: + +```bash +# Profile weak scaling performance at different node counts +./submit.sh --likwid -N 2 -n 2 -c 64 -- --num-humans 100 # Baseline profiling (2 COSMA5 nodes) +./submit.sh --likwid -N 3 -n 3 -c 64 -- --num-humans 122 # 3-node scaling analysis (max COSMA5) +./submit.sh --vtune -N 8 -n 8 -c 16 -p cosma -- --num-humans 200 # 8-node analysis on COSMA legacy +./submit.sh --vtune -N 18 -n 18 -c 16 -p cosma -- --num-humans 300 # 18-node analysis on COSMA legacy +``` + +**Input File Naming Convention:** + +The [`gen.sh`](examples/HumanMobility/gen.sh) script generates files with names based on the `--num-humans` parameter: +- `humans-rivers-{NUM_HUMANS}.hdf5` (or `humans-rivers-{NUM_HUMANS}-cosma.hdf5` on COSMA partition) +- `river-rivers-{NUM_HUMANS}.hdf5` (or `river-rivers-{NUM_HUMANS}-cosma.hdf5` on COSMA partition) + +The [`run.sh`](examples/HumanMobility/run.sh) script automatically uses the correct input files when passed the `--num-humans` parameter, ensuring that: +- 2-node runs use the 100×100 human configuration +- 3-node runs use the 122×122 human configuration +- 8-node runs use the 200×200 human configuration +- 18-node runs use the 300×300 human configuration + +**Expected Results:** +- Execution time should remain approximately constant across all configurations +- Memory usage per node should remain consistent (proportional to local problem size) +- Communication overhead should grow modestly with node count +- Each configuration maintains the same work per computational unit + +**Key Points for Weak Scaling:** +1. **Generate all input files first** before running scaling tests +2. **Use `--num-humans` parameter** to specify which input files to use +3. **Ensure domain size scales with human count** (maintain constant density) +4. **Verify file naming consistency** across partitions (COSMA vs COSMA5) + +_Note: COSMA5 is limited to 3 nodes maximum, so extended weak scaling studies should use the legacy COSMA partition which has ~160 nodes available. The input file generation step ensures that each node configuration has appropriately sized problems while maintaining constant work density._ + +### Visualisation of performance results + +The performance analysis results can be visualized using the `pa_vis.py` script, which creates comprehensive PDF reports showing memory usage, CPU performance, and thread utilization across MPI ranks. + +#### Usage + +```bash +python pa_vis.py -d +``` + +**Command-line options:** +- `-d, --directory`: Directory containing performance log files (default: current directory) +- `--use-balance-logs`: Force use of `rank_*_balance.log` files +- `--use-dat-files`: Force use of `*_report-rank*-step*.dat` files +- `--use-thread-files`: Force use of `thread_info_MPI-step*.dat` files + +The script automatically detects and processes the following data sources (in order of preference): +1. **Balance logs**: `rank_memory_balance.log` and `rank_cpu_balance.log` +2. **Individual report files**: `memuse_report-rank*-step*.dat` and `mpiuse_report-rank*-step*.dat` +3. **Thread timing data**: `thread_info_MPI-step*.dat` or `thread_info-step*.dat` + +#### Output + +The tool generates a two-page PDF report: + +**Page 1: System-level Performance** +- Memory usage across MPI ranks (average and peak) +- CPU time distribution and parallel efficiency +- Load balancing metrics + +**Page 2: Thread-level Analysis** +- Individual thread utilization by rank +- Thread balance ratios within each rank +- Thread efficiency visualization + +#### Example Usage + +For a simulation run with directory `data-rivers-1-4-4-cosma`: + +```bash +python pa_vis.py -d data-rivers-1-4-4-cosma +``` + +**Sample Output:** +``` +Detected partition: cosma +Searching for performance files in: .../SWIFT/examples/HumanMobility/data-rivers-1-4-4-cosma +Detected 1 nodes from directory name +Output will be saved as: pa_vis-1-4-4-cosma.pdf +Using balance log files: .../rank_memory_balance.log, .../rank_cpu_balance.log +Using MPI thread info files: 100 files found +Found 4 ranks and 17 balance steps + +============================================================ +PERFORMANCE ANALYSIS SUMMARY +============================================================ +Simulation: 17 steps, 4 MPI ranks +Memory & CPU steps analyzed: 17, 25, 33, 41, 49, 57, 65, 75, 90, 126, 190, 252, 370, 524, 635, 745, 934 +Thread steps analyzed: 2 to 992 (100 total steps) + +MEMORY USAGE: + Rank 0: Avg = 1196.63 GB, Max = 1242.03 GB + Rank 1: Avg = 1167.23 GB, Max = 1201.75 GB + Rank 2: Avg = 1165.93 GB, Max = 1233.33 GB + Rank 3: Avg = 1151.01 GB, Max = 1208.53 GB + Overall average: 1170.20 GB + Memory balance: 1.4% + +CPU USAGE: + Rank 0: Total = 81.9 seconds + Rank 1: Total = 81.0 seconds + Rank 2: Total = 81.0 seconds + Rank 3: Total = 79.8 seconds + Overall total: 323.8 seconds + Parallel efficiency: 98.8% + Load balance ratio: 0.97 + +INDIVIDUAL THREAD USAGE: + Rank 0-3 Thread 0-3: 97.7 - 246.8 s (per thread) + Total threads active: 16 + Overall total thread time: 2114.4 s + Thread balance ratio: 0.40 + +Output saved to: pa_vis-1-4-4-cosma.pdf +============================================================ +``` + +The output PDF filename automatically includes the run configuration (e.g., `pa_vis-1-4-4-cosma.pdf`) for easy identification and comparison across different configurations. \ No newline at end of file diff --git a/examples/HumanMobility/doc/paws.md b/examples/HumanMobility/doc/paws.md new file mode 100644 index 000000000..ae5ea6934 --- /dev/null +++ b/examples/HumanMobility/doc/paws.md @@ -0,0 +1,1070 @@ +# Performance analysis + +Application: Human Mobility + +Parameters: + +- Number of people: 10k → 1M humans +- Area of territory: 100 sq km → 10000 sq km + +## Setup + +* Cosma + +## Benchmarks + +### Memory bandwidth (MByte/s) on a single core +``` +likwid-bench -t triad -W S0:2GB:1 +``` +``` +==== +Starting job ... +... +LIKWID MICRO BENCHMARK +Test: triad +... +Using 1 work groups +Using 1 threads +... +Cycles: 3591308877 +CPU Clock: 2599987191 +Cycle Clock: 2599987191 +Time: 1.381279e+00 sec +Iterations: 10 +Iterations per thread: 10 +Inner loop executions: 15625000 +Size (Byte): 2000000000 +Size per thread: 2000000000 +Number of Flops: 1250000000 +MFlops/s: 904.96 +Data volume (Byte): 20000000000 +MByte/s: 14479.33 +Cycles per update: 5.746094 +Cycles per cacheline: 45.968754 +Loads per update: 3 +Stores per update: 1 +Load bytes per element: 24 +Store bytes per elem.: 8 +Load/store ratio: 3.00 +Instructions: 2968750016 +UOPs: 4687500000 +-------------------------------------------------------------------------------- +``` + +### Memory bandwidth (MByte/s) on a full node +``` +likwid-bench -t triad -W N:2GB:16 +``` +``` +==== +Starting job ... +... +LIKWID MICRO BENCHMARK +Test: triad +... +Using 1 work groups +Using 16 threads +... +Cycles: 25008707817 +CPU Clock: 2599930277 +Cycle Clock: 2599930277 +Time: 9.618992e+00 sec +Iterations: 2048 +Iterations per thread: 128 +Inner loop executions: 976562 +Size (Byte): 1999998976 +Size per thread: 124999936 +Number of Flops: 15999991808 +MFlops/s: 1663.38 +Data volume (Byte): 255999868928 +MByte/s: 26614.00 +Cycles per update: 3.126090 +Cycles per cacheline: 25.008721 +Loads per update: 3 +Stores per update: 1 +Load bytes per element: 24 +Store bytes per elem.: 8 +Load/store ratio: 3.00 +Instructions: 37999980560 +UOPs: 59999969280 +-------------------------------------------------------------------------------- +``` + +### Peak floating-point performance (MFLOPS) on a single core +``` +likwid-bench -t peakflops -w S0:10000MB:1 +``` +``` +==== +Starting job ... +... +LIKWID MICRO BENCHMARK +Test: peakflops +... +Using 1 work groups +Using 1 threads +... +Cycles: 105343024434 +CPU Clock: 2599963429 +Cycle Clock: 2599963429 +Time: 4.051712e+01 sec +Iterations: 10 +Iterations per thread: 10 +Inner loop executions: 1250000000 +Size (Byte): 10000000000 +Size per thread: 10000000000 +Number of Flops: 200000000000 +MFlops/s: 4936.19 +Data volume (Byte): 100000000000 +MByte/s: 2468.09 +Cycles per update: 8.427442 +Cycles per cacheline: 67.419536 +Loads per update: 1 +Stores per update: 0 +Load bytes per element: 8 +Store bytes per elem.: 0 +Instructions: 250000000032 +UOPs: 237500000000 +-------------------------------------------------------------------------------- +``` + +### Peak floating-point performance (MFLOPS) on a full node +``` +likwid-bench -t peakflops -w N:10000MB:16 +``` +``` +==== +Starting job ... +... +LIKWID MICRO BENCHMARK +Test: peakflops +... +Using 1 work groups +Using 16 threads +... +Cycles: 11756247712 +CPU Clock: 2599984198 +Cycle Clock: 2599984198 +Time: 4.521661e+00 sec +Iterations: 160 +Iterations per thread: 10 +Inner loop executions: 78125000 +Size (Byte): 10000000000 +Size per thread: 625000000 +Number of Flops: 200000000000 +MFlops/s: 44231.53 +Data volume (Byte): 100000000000 +MByte/s: 22115.77 +Cycles per update: 0.940500 +Cycles per cacheline: 7.523999 +Loads per update: 1 +Stores per update: 0 +Load bytes per element: 8 +Store bytes per elem.: 0 +Instructions: 250000000032 +UOPs: 237500000000 +-------------------------------------------------------------------------------- +``` + +## Core (caches) + +### Serial + +#### Memory performance (`--enable-debug`) + +##### 100 x 100 humans on 10 x 10 km for minimum(1000 steps, 10 minutes): +``` +likwid-perfctr -f -C 0 -g MEM swift -A -s -g -G --hm-river --hm-randomwalk --threads=1 -n 1000 humanMobility.yml +``` +``` +-------------------------------------------------------------------------------- +Group 1: MEM ++-----------------------+----------+--------------+ +| Event | Counter | HWThread 0 | ++-----------------------+----------+--------------+ +| INSTR_RETIRED_ANY | FIXC0 | 123986843446 | +| CPU_CLK_UNHALTED_CORE | FIXC1 | 70409818611 | +| CPU_CLK_UNHALTED_REF | FIXC2 | 37073751360 | +| TOPDOWN_SLOTS | FIXC3 | 422458260954 | +| CAS_COUNT_RD | MBOX0C0 | 2104365 | +| CAS_COUNT_WR | MBOX0C1 | 1941294 | +| CAS_COUNT_RD | MBOX1C0 | 2034649 | +| CAS_COUNT_WR | MBOX1C1 | 1885696 | +| CAS_COUNT_RD | MBOX2C0 | 1971302 | +| CAS_COUNT_WR | MBOX2C1 | 1827391 | +| CAS_COUNT_RD | MBOX3C0 | 2004834 | +| CAS_COUNT_WR | MBOX3C1 | 1856898 | +| CAS_COUNT_RD | MBOX4C0 | 2103657 | +| CAS_COUNT_WR | MBOX4C1 | 1938628 | +| CAS_COUNT_RD | MBOX5C0 | 2029864 | +| CAS_COUNT_WR | MBOX5C1 | 1879532 | +| CAS_COUNT_RD | MBOX6C0 | 1974026 | +| CAS_COUNT_WR | MBOX6C1 | 1830926 | +| CAS_COUNT_RD | MBOX7C0 | 2019975 | +| CAS_COUNT_WR | MBOX7C1 | 1867911 | ++-----------------------+----------+--------------+ + ++-----------------------------------+------------+ +| Metric | HWThread 0 | ++-----------------------------------+------------+ +| Runtime (RDTSC) [s] | 19.4184 | +| Runtime unhalted [s] | 35.2053 | +| Clock [MHz] | 3798.3176 | +| CPI | 0.5679 | +| Memory read bandwidth [MBytes/s] | 53.5334 | +| Memory read data volume [GBytes] | 1.0395 | +| Memory write bandwidth [MBytes/s] | 49.5309 | +| Memory write data volume [GBytes] | 0.9618 | +| Memory bandwidth [MBytes/s] | 103.0643 | +| Memory data volume [GBytes] | 2.0013 | ++-----------------------------------+------------+ +``` + +##### 1000 x 1000 humans on 100 x 100 km for minimum(100 steps, 10 minutes): +``` +likwid-perfctr -f -C 0 -g MEM swift -A -s -g -G --hm-river --hm-randomwalk --threads=16 -n 100 humanMobility.yml +``` +``` +-------------------------------------------------------------------------------- +Group 1: MEM ++-----------------------+----------+----------------+ +| Event | Counter | HWThread 0 | ++-----------------------+----------+----------------+ +| INSTR_RETIRED_ANY | FIXC0 | 10078700900771 | +| CPU_CLK_UNHALTED_CORE | FIXC1 | 4651632179068 | +| CPU_CLK_UNHALTED_REF | FIXC2 | 2450068924800 | +| TOPDOWN_SLOTS | FIXC3 | 27909716623362 | +| CAS_COUNT_RD | MBOX0C0 | 296253668 | +| CAS_COUNT_WR | MBOX0C1 | 198353712 | +| CAS_COUNT_RD | MBOX1C0 | 295934061 | +| CAS_COUNT_WR | MBOX1C1 | 198108469 | +| CAS_COUNT_RD | MBOX2C0 | 296181293 | +| CAS_COUNT_WR | MBOX2C1 | 198068319 | +| CAS_COUNT_RD | MBOX3C0 | 296381234 | +| CAS_COUNT_WR | MBOX3C1 | 198231128 | +| CAS_COUNT_RD | MBOX4C0 | 296422014 | +| CAS_COUNT_WR | MBOX4C1 | 198429254 | +| CAS_COUNT_RD | MBOX5C0 | 296071633 | +| CAS_COUNT_WR | MBOX5C1 | 198181758 | +| CAS_COUNT_RD | MBOX6C0 | 296372650 | +| CAS_COUNT_WR | MBOX6C1 | 198253434 | +| CAS_COUNT_RD | MBOX7C0 | 296656051 | +| CAS_COUNT_WR | MBOX7C1 | 198468244 | ++-----------------------+----------+----------------+ + ++-----------------------------------+------------+ +| Metric | HWThread 0 | ++-----------------------------------+------------+ +| Runtime (RDTSC) [s] | 1229.3598 | +| Runtime unhalted [s] | 2325.8498 | +| Clock [MHz] | 3797.0889 | +| CPI | 0.4615 | +| Memory read bandwidth [MBytes/s] | 123.3955 | +| Memory read data volume [GBytes] | 151.6974 | +| Memory write bandwidth [MBytes/s] | 82.5715 | +| Memory write data volume [GBytes] | 101.5100 | +| Memory bandwidth [MBytes/s] | 205.9670 | +| Memory data volume [GBytes] | 253.2075 | ++-----------------------------------+------------+ +``` + +##### 1000 x 1000 humans on 100 x 100 km for minimum, multiple rivers (1000 steps, 10 minutes): +``` +likwid-perfctr -f -C 0 -g MEM swift -A -s -g -G --hm-river --hm-randomwalk --threads=16 -n 100 humanMobility.yml +``` +``` +-------------------------------------------------------------------------------- +Group 1: MEM ++-----------------------+---------+----------------+ +| Event | Counter | HWThread 0 | ++-----------------------+---------+----------------+ +| INSTR_RETIRED_ANY | FIXC0 | 16237189087458 | +| CPU_CLK_UNHALTED_CORE | FIXC1 | 13024552770775 | +| CPU_CLK_UNHALTED_REF | FIXC2 | 10265924898034 | +| CAS_COUNT_RD | MBOX0C0 | 4634215368 | +| CAS_COUNT_WR | MBOX0C1 | 1900293496 | +| CAS_COUNT_RD | MBOX1C0 | 3896604436 | +| CAS_COUNT_WR | MBOX1C1 | 1068569638 | +| CAS_COUNT_RD | MBOX2C0 | 3801181672 | +| CAS_COUNT_WR | MBOX2C1 | 1139534217 | +| CAS_COUNT_RD | MBOX3C0 | 3903990616 | +| CAS_COUNT_WR | MBOX3C1 | 1066476734 | ++-----------------------+---------+----------------+ + ++-----------------------------------+------------+ +| Metric | HWThread 0 | ++-----------------------------------+------------+ +| Runtime (RDTSC) [s] | 3994.6320 | +| Runtime unhalted [s] | 5009.5327 | +| Clock [MHz] | 3298.6052 | +| CPI | 0.8021 | +| Memory read bandwidth [MBytes/s] | 260.1250 | +| Memory read data volume [GBytes] | 1039.1035 | +| Memory write bandwidth [MBytes/s] | 82.9092 | +| Memory write data volume [GBytes] | 331.1919 | +| Memory bandwidth [MBytes/s] | 343.0342 | +| Memory data volume [GBytes] | 1370.2954 | ++-----------------------------------+------------+ +``` + +#### Floating-point performance (`--enable-debug`) + +##### 100 x 100 humans on 10 x 10 km for minimum(1000 steps, 10 minutes): +``` +likwid-perfctr -f -C 0 -g FLOPS_DP swift -A -s -g -G --hm-river --hm-randomwalk --threads=1 -n 1000 humanMobility.yml +``` +``` +-------------------------------------------------------------------------------- +Group 1: FLOPS_DP ++------------------------------------------+---------+--------------+ +| Event | Counter | HWThread 0 | ++------------------------------------------+---------+--------------+ +| INSTR_RETIRED_ANY | FIXC0 | 143861628268 | +| CPU_CLK_UNHALTED_CORE | FIXC1 | 79237524584 | +| CPU_CLK_UNHALTED_REF | FIXC2 | 41722922320 | +| TOPDOWN_SLOTS | FIXC3 | 475423666740 | +| FP_ARITH_INST_RETIRED_128B_PACKED_DOUBLE | PMC0 | 418829623 | +| FP_ARITH_INST_RETIRED_SCALAR_DOUBLE | PMC1 | 1872322584 | +| FP_ARITH_INST_RETIRED_256B_PACKED_DOUBLE | PMC2 | 295344149 | +| FP_ARITH_INST_RETIRED_512B_PACKED_DOUBLE | PMC3 | 0 | ++------------------------------------------+---------+--------------+ + ++----------------------+------------+ +| Metric | HWThread 0 | ++----------------------+------------+ +| Runtime (RDTSC) [s] | 21.9525 | +| Runtime unhalted [s] | 39.6194 | +| Clock [MHz] | 3798.2077 | +| CPI | 0.5508 | +| DP [MFLOP/s] | 177.2623 | +| AVX DP [MFLOP/s] | 53.8150 | +| AVX512 DP [MFLOP/s] | 0 | +| Packed [MUOPS/s] | 32.5326 | +| Scalar [MUOPS/s] | 85.2895 | +| Vectorization ratio | 27.6116 | ++----------------------+------------+ +``` + +##### 1000 x 1000 humans on 100 x 100 km for minimum(100 steps, 10 minutes): +``` +likwid-perfctr -f -C 0 -g FLOPS_DP swift -A -s -g -G --hm-river --hm-randomwalk --threads=16 -n 100 humanMobility.yml +``` +``` +-------------------------------------------------------------------------------- +Group 1: FLOPS_DP ++------------------------------------------+---------+----------------+ +| Event | Counter | HWThread 0 | ++------------------------------------------+---------+----------------+ +| INSTR_RETIRED_ANY | FIXC0 | 9763512748911 | +| CPU_CLK_UNHALTED_CORE | FIXC1 | 4519527714286 | +| CPU_CLK_UNHALTED_REF | FIXC2 | 2380559070800 | +| TOPDOWN_SLOTS | FIXC3 | 27117099211224 | +| FP_ARITH_INST_RETIRED_128B_PACKED_DOUBLE | PMC0 | 12350095127 | +| FP_ARITH_INST_RETIRED_SCALAR_DOUBLE | PMC1 | 13593594077 | +| FP_ARITH_INST_RETIRED_256B_PACKED_DOUBLE | PMC2 | 10064623960 | +| FP_ARITH_INST_RETIRED_512B_PACKED_DOUBLE | PMC3 | 0 | ++------------------------------------------+---------+----------------+ + ++----------------------+------------+ +| Metric | HWThread 0 | ++----------------------+------------+ +| Runtime (RDTSC) [s] | 1195.7233 | +| Runtime unhalted [s] | 2259.8068 | +| Clock [MHz] | 3796.9584 | +| CPI | 0.4629 | +| DP [MFLOP/s] | 65.6944 | +| AVX DP [MFLOP/s] | 33.6687 | +| AVX512 DP [MFLOP/s] | 0 | +| Packed [MUOPS/s] | 18.7457 | +| Scalar [MUOPS/s] | 11.3685 | +| Vectorization ratio | 62.2487 | ++----------------------+------------+ +``` + +##### 1000 x 1000 humans on 100 x 100 km for minimum, multiple rivers (1000 steps, 10 minutes): +``` +likwid-perfctr -f -C 0 -g FLOPS_DP swift -A -s -g -G --hm-river --hm-randomwalk --threads=16 -n 100 humanMobility.yml +``` +``` +-------------------------------------------------------------------------------- +Group 1: FLOPS_DP ++--------------------------------------+---------+----------------+ +| Event | Counter | HWThread 0 | ++--------------------------------------+---------+----------------+ +| INSTR_RETIRED_ANY | FIXC0 | 16579362600938 | +| CPU_CLK_UNHALTED_CORE | FIXC1 | 13307840039017 | +| CPU_CLK_UNHALTED_REF | FIXC2 | 10488495793516 | +| FP_COMP_OPS_EXE_SSE_FP_PACKED_DOUBLE | PMC0 | 52287698182 | +| FP_COMP_OPS_EXE_SSE_FP_SCALAR_DOUBLE | PMC1 | 69433493065 | +| SIMD_FP_256_PACKED_DOUBLE | PMC2 | 82768760959 | ++--------------------------------------+---------+----------------+ + ++-------------------------+------------+ +| Metric | HWThread 0 | ++-------------------------+------------+ +| Runtime (RDTSC) [s] | 4080.0487 | +| Runtime unhalted [s] | 5118.4970 | +| Clock [MHz] | 3298.8265 | +| CPI | 0.8027 | +| DP [MFLOP/s] | 123.7936 | +| AVX DP [MFLOP/s] | 81.1449 | +| Packed [MUOPS/s] | 33.1017 | +| Scalar [MUOPS/s] | 17.0178 | +| Vectorization ratio [%] | 66.0455 | ++-------------------------+------------+ +``` + +#### Comparison wrt scaling and with microbenchmarks + +--- + +1. **Scaling: 100x100 Humans (10km x 10km) vs 1000x1000 Humans (100km x 100km)** + +**Memory Metrics** (`MEM` group) + +| Metric | 100x100 Humans | 1000x1000 Humans | Change | +|-------------------------------|----------------|------------------|-----------------------| +| Runtime (RDTSC) [s] | 19.42 | 1229.36 | ~63.3x increase | +| Memory BW [MB/s] | 103.06 | 205.97 | ~2x increase | +| Memory Data Volume [GB] | 2.00 | 253.21 | ~126.6x increase | +| CPI | 0.5679 | 0.4615 | ~18.7% better | + +**Floating-Point Metrics** (`FLOPS_DP` group) + +| Metric | 100x100 Humans | 1000x1000 Humans | Change | +|------------------|----------------|------------------|-----------------------| +| DP MFLOP/s | 177.26 | 65.69 | ~63% decrease | +| Vectorization [%]| 27.61 | 62.25 | ~2.25x improvement | +| CPI | 0.5508 | 0.4629 | ~16% improvement | + +--- + +2. **Comparison with `likwid-bench triad`** + +| Metric | SWIFT (best case) | likwid-bench triad | Ratio | +|----------------|------------------|--------------------|------------| +| Memory BW | 205.97 MB/s | 26614.00 MB/s | ~0.77% | +| Read/Write Ratio | 1.1:1 | 3:1 | Lower | +| CPI | 0.4615 | 3.13 | ~6.8x better | + +--- + +3. **Comparison with `likwid-bench peakflops`** + +| Metric | SWIFT (best case) | likwid-bench peakflops (single core) | likwid-bench peakflops (full node) | Ratio (single core) | Ratio (full node) | +|----------------|------------------|--------------------------------------|------------------------------------|---------------------|-------------------| +| MFLOP/s | 177.26 | 4936.19 | 44231.53 | ~3.6% | ~0.4% | +| Vectorization | 62.25% | ~100% | ~100% | ~0.62x | ~0.62x | +| CPI | 0.4629 | 8.43 | 0.94 | ~18x better | ~2x better | + +--- + +**Summary Table** + +| Metric | SWIFT (large, best case) | triad (single core) | triad (full node) | peakflops (single core) | peakflops (full node) | +|-----------------------|-------------------------|---------------------|-------------------|-------------------------|-----------------------| +| Memory BW [MB/s] | 205.97 | 14479.33 | 26614.00 | 2468.09 | 22115.77 | +| DP MFLOP/s | 65.69 | (not measured) | (not measured) | 4936.19 | 44231.53 | +| CPI | 0.4615 | 3.13 | 3.13 | 8.43 | 0.94 | +| Vectorization [%] | 62.25 | (not measured) | (not measured) | ~100 | ~100 | + +--- + +**Key Takeaways** + +- **Scaling:** Memory bandwidth and data volume increase with problem size, but bandwidth is still far from hardware peak. +- **Compared to `triad`:** SWIFT achieves only a small fraction of the streaming bandwidth, with a more balanced (less streaming) access pattern and better CPI. +- **Compared to `peakflops`:** SWIFT achieves only ~3.6% of single-core peak FLOPS, and about 0.4% of full-node `peakflops` MFLOPS, with much lower vectorization. +- **Optimization potential:** There is significant headroom for improving both memory access patterns and vectorization in SWIFT. + +--- + +### Parallel + +#### Memory performance (`--enable-debug`, 4 MPI ranks, 4 OMP threads) + +##### 100 x 100 humans on 10 x 10 km for minimum(10000 steps, 10 minutes): + +``` +likwid-mpirun -np 4 -t 4 -omp intel -g MEM -- swift_mpi --threads=4 -A -s -g -G --hm-river --hm-randomwalk -n 10000 ${HUMANMOBILITY}.yml +``` +``` +Group: 1 ++-----------------------+---------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+ +| Event | Counter | m5228:0:0 | m5228:0:1 | m5228:0:2 | m5228:0:3 | m5228:1:4 | m5228:1:5 | m5228:1:6 | m5228:1:7 | m5228:2:8 | m5228:2:9 | m5228:2:10 | m5228:2:11 | m5228:3:12 | m5228:3:13 | m5228:3:14 | m5228:3:15 | ++-----------------------+---------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+ +| INSTR_RETIRED_ANY | FIXC0 | 341001383176 | 94476288753 | 108228489509 | 157139864508 | 359764424707 | 92216965479 | 104581778267 | 150788089375 | 371625089799 | 94591782131 | 109177270245 | 154429978651 | 337094080890 | 96098187390 | 110280986563 | 159900454880 | +| CPU_CLK_UNHALTED_CORE | FIXC1 | 280895787571 | 108450104598 | 74982138021 | 140211569134 | 288127266919 | 101224781427 | 72250116148 | 132304678131 | 293130918907 | 104687083572 | 73625866015 | 134097259742 | 286173688419 | 109087093895 | 75794547982 | 142937378290 | +| CPU_CLK_UNHALTED_REF | FIXC2 | 234079183520 | 92927087344 | 64614831190 | 120183334544 | 240140408820 | 86879553592 | 62273975608 | 113511073546 | 242431346300 | 90063364742 | 63294626902 | 115109635966 | 236550605798 | 93637848070 | 65134683822 | 122506849556 | +| CAS_COUNT_RD | MBOX0C0 | 395104151 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 391047867 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| CAS_COUNT_WR | MBOX0C1 | 323044630 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 301155310 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| CAS_COUNT_RD | MBOX1C0 | 378297602 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 403500528 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| CAS_COUNT_WR | MBOX1C1 | 298478851 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 307279648 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| CAS_COUNT_RD | MBOX2C0 | 389514168 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 511476132 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| CAS_COUNT_WR | MBOX2C1 | 309200166 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 363461718 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| CAS_COUNT_RD | MBOX3C0 | 372696559 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 463002788 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| CAS_COUNT_WR | MBOX3C1 | 296464156 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 306713669 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | ++-----------------------+---------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+ + ++----------------------------+---------+---------------+-------------+--------------+--------------+ +| Event | Counter | Sum | Min | Max | Avg | ++----------------------------+---------+---------------+-------------+--------------+--------------+ +| INSTR_RETIRED_ANY STAT | FIXC0 | 2841395114323 | 92216965479 | 371625089799 | 1.775872e+11 | +| CPU_CLK_UNHALTED_CORE STAT | FIXC1 | 2417980278771 | 72250116148 | 293130918907 | 1.511238e+11 | +| CPU_CLK_UNHALTED_REF STAT | FIXC2 | 2043338409320 | 62273975608 | 242431346300 | 1.277087e+11 | +| CAS_COUNT_RD STAT | MBOX0C0 | 786152018 | 0 | 395104151 | 4.913450e+07 | +| CAS_COUNT_WR STAT | MBOX0C1 | 624199940 | 0 | 323044630 | 3.901250e+07 | +| CAS_COUNT_RD STAT | MBOX1C0 | 781798130 | 0 | 403500528 | 4.886238e+07 | +| CAS_COUNT_WR STAT | MBOX1C1 | 605758499 | 0 | 307279648 | 3.785991e+07 | +| CAS_COUNT_RD STAT | MBOX2C0 | 900990300 | 0 | 511476132 | 5.631189e+07 | +| CAS_COUNT_WR STAT | MBOX2C1 | 672661884 | 0 | 363461718 | 4.204137e+07 | +| CAS_COUNT_RD STAT | MBOX3C0 | 835699347 | 0 | 463002788 | 5.223121e+07 | +| CAS_COUNT_WR STAT | MBOX3C1 | 603177825 | 0 | 306713669 | 3.769861e+07 | ++----------------------------+---------+---------------+-------------+--------------+--------------+ + ++-----------------------------------+-----------+-----------+-----------+-----------+-----------+-----------+-----------+-----------+-----------+-----------+------------+------------+------------+------------+------------+------------+ +| Metric | m5228:0:0 | m5228:0:1 | m5228:0:2 | m5228:0:3 | m5228:1:4 | m5228:1:5 | m5228:1:6 | m5228:1:7 | m5228:2:8 | m5228:2:9 | m5228:2:10 | m5228:2:11 | m5228:3:12 | m5228:3:13 | m5228:3:14 | m5228:3:15 | ++-----------------------------------+-----------+-----------+-----------+-----------+-----------+-----------+-----------+-----------+-----------+-----------+------------+------------+------------+------------+------------+------------+ +| Runtime (RDTSC) [s] | 136.6862 | 136.6862 | 136.6862 | 136.6862 | 136.6821 | 136.6821 | 136.6821 | 136.6821 | 136.6847 | 136.6847 | 136.6847 | 136.6847 | 136.6827 | 136.6827 | 136.6827 | 136.6827 | +| Runtime unhalted [s] | 108.0391 | 41.7124 | 28.8399 | 53.9286 | 110.8196 | 38.9331 | 27.7889 | 50.8871 | 112.7437 | 40.2646 | 28.3179 | 51.5763 | 110.0680 | 41.9570 | 29.1521 | 54.9765 | +| Clock [MHz] | 3119.9445 | 3034.2551 | 3017.1023 | 3033.2209 | 3119.5129 | 3029.2627 | 3016.4743 | 3030.4327 | 3143.7093 | 3022.1391 | 3024.3572 | 3028.8498 | 3145.3893 | 3028.9404 | 3025.4810 | 3033.5714 | +| CPI | 0.8237 | 1.1479 | 0.6928 | 0.8923 | 0.8009 | 1.0977 | 0.6908 | 0.8774 | 0.7888 | 1.1067 | 0.6744 | 0.8683 | 0.8489 | 1.1352 | 0.6873 | 0.8939 | +| Memory read bandwidth [MBytes/s] | 719.0133 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 828.3133 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| Memory read data volume [GBytes] | 98.2792 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 113.2177 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| Memory write bandwidth [MBytes/s] | 574.6009 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 598.6849 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| Memory write data volume [GBytes] | 78.5400 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 81.8311 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| Memory bandwidth [MBytes/s] | 1293.6143 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1426.9981 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| Memory data volume [GBytes] | 176.8192 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 195.0488 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | ++-----------------------------------+-----------+-----------+-----------+-----------+-----------+-----------+-----------+-----------+-----------+-----------+------------+------------+------------+------------+------------+------------+ + ++----------------------------------------+------------+-----------+-----------+-----------+-----------+-----------+-----------+ +| Metric | Sum | Min | Max | Avg | %ile 25 | %ile 50 | %ile 75 | ++----------------------------------------+------------+-----------+-----------+-----------+-----------+-----------+-----------+ +| Runtime (RDTSC) [s] STAT | 2186.9428 | 136.6821 | 136.6862 | 136.6839 | 136.6821 | 136.6827 | 136.6847 | +| Runtime unhalted [s] STAT | 930.0048 | 27.7889 | 112.7437 | 58.1253 | 112.7437 | 29.1521 | 41.9570 | +| Clock [MHz] STAT | 48852.6429 | 3016.4743 | 3145.3893 | 3053.2902 | 3024.3572 | 3029.2627 | 3034.2551 | +| CPI STAT | 14.0270 | 0.6744 | 1.1479 | 0.8767 | 0.6928 | 0.8489 | 0.8939 | +| Memory read bandwidth [MBytes/s] STAT | 1547.3266 | 0 | 828.3133 | 96.7079 | 0 | 0 | 0 | +| Memory read data volume [GBytes] STAT | 211.4969 | 0 | 113.2177 | 13.2186 | 0 | 0 | 0 | +| Memory write bandwidth [MBytes/s] STAT | 1173.2858 | 0 | 598.6849 | 73.3304 | 0 | 0 | 0 | +| Memory write data volume [GBytes] STAT | 160.3711 | 0 | 81.8311 | 10.0232 | 0 | 0 | 0 | +| Memory bandwidth [MBytes/s] STAT | 2720.6124 | 0 | 1426.9981 | 170.0383 | 0 | 0 | 0 | +| Memory data volume [GBytes] STAT | 371.8680 | 0 | 195.0488 | 23.2417 | 0 | 0 | 0 | ++----------------------------------------+------------+-----------+-----------+-----------+-----------+-----------+-----------+ +``` + +##### 1000 x 1000 humans on 100 x 100 km for minimum(1000 steps, 10 minutes): + +``` +likwid-mpirun -np 4 -t 4 -omp intel -g MEM -- swift_mpi --threads=4 -A -s -g -G --hm-river --hm-randomwalk -n 10000 ${HUMANMOBILITY}.yml +``` +``` +Group: 1 ++-----------------------+---------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+ +| Event | Counter | m5237:0:0 | m5237:0:1 | m5237:0:2 | m5237:0:3 | m5237:1:4 | m5237:1:5 | m5237:1:6 | m5237:1:7 | m5237:2:8 | m5237:2:9 | m5237:2:10 | m5237:2:11 | m5237:3:12 | m5237:3:13 | m5237:3:14 | m5237:3:15 | ++-----------------------+---------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+ +| INSTR_RETIRED_ANY | FIXC0 | 935027159945 | 918078869462 | 945972539732 | 957959870956 | 896847519524 | 874744379907 | 924260839133 | 927599423107 | 916968570718 | 890387375529 | 933863665539 | 943080011762 | 903224228888 | 883709778741 | 928253499166 | 933640852265 | +| CPU_CLK_UNHALTED_CORE | FIXC1 | 804755146770 | 779172644940 | 781786388839 | 789189897021 | 768855237917 | 744265028808 | 764818386042 | 763329436089 | 787163890777 | 758183098203 | 775728499290 | 775663816515 | 773399863303 | 748645688585 | 767735222132 | 764163093009 | +| CPU_CLK_UNHALTED_REF | FIXC2 | 696778006886 | 675063194728 | 677139994726 | 683414274686 | 665560928110 | 644816364608 | 662459649124 | 661041261192 | 680841627024 | 656554391870 | 671170753370 | 671005159318 | 669071116116 | 648364992652 | 664244851062 | 660990384080 | +| CAS_COUNT_RD | MBOX0C0 | 1303101073 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1173952283 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| CAS_COUNT_WR | MBOX0C1 | 604088681 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 516028925 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| CAS_COUNT_RD | MBOX1C0 | 1253823219 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1181399098 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| CAS_COUNT_WR | MBOX1C1 | 518406987 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 493431066 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| CAS_COUNT_RD | MBOX2C0 | 1249469599 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1279012528 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| CAS_COUNT_WR | MBOX2C1 | 547663411 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 570219086 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| CAS_COUNT_RD | MBOX3C0 | 1251631196 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1274434211 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| CAS_COUNT_WR | MBOX3C1 | 517600181 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 493283369 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | ++-----------------------+---------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+ + ++----------------------------+---------+----------------+--------------+--------------+--------------+ +| Event | Counter | Sum | Min | Max | Avg | ++----------------------------+---------+----------------+--------------+--------------+--------------+ +| INSTR_RETIRED_ANY STAT | FIXC0 | 14713618584374 | 874744379907 | 957959870956 | 9.196012e+11 | +| CPU_CLK_UNHALTED_CORE STAT | FIXC1 | 12346855338240 | 744265028808 | 804755146770 | 771678458640 | +| CPU_CLK_UNHALTED_REF STAT | FIXC2 | 10688516949552 | 644816364608 | 696778006886 | 668032309347 | +| CAS_COUNT_RD STAT | MBOX0C0 | 2477053356 | 0 | 1303101073 | 1.548158e+08 | +| CAS_COUNT_WR STAT | MBOX0C1 | 1120117606 | 0 | 604088681 | 7.000735e+07 | +| CAS_COUNT_RD STAT | MBOX1C0 | 2435222317 | 0 | 1253823219 | 1.522014e+08 | +| CAS_COUNT_WR STAT | MBOX1C1 | 1011838053 | 0 | 518406987 | 6.323988e+07 | +| CAS_COUNT_RD STAT | MBOX2C0 | 2528482127 | 0 | 1279012528 | 1.580301e+08 | +| CAS_COUNT_WR STAT | MBOX2C1 | 1117882497 | 0 | 570219086 | 6.986766e+07 | +| CAS_COUNT_RD STAT | MBOX3C0 | 2526065407 | 0 | 1274434211 | 1.578791e+08 | +| CAS_COUNT_WR STAT | MBOX3C1 | 1010883550 | 0 | 517600181 | 6.318022e+07 | ++----------------------------+---------+----------------+--------------+--------------+--------------+ + ++-----------------------------------+-----------+-----------+-----------+-----------+-----------+-----------+-----------+-----------+-----------+-----------+------------+------------+------------+------------+------------+------------+ +| Metric | m5237:0:0 | m5237:0:1 | m5237:0:2 | m5237:0:3 | m5237:1:4 | m5237:1:5 | m5237:1:6 | m5237:1:7 | m5237:2:8 | m5237:2:9 | m5237:2:10 | m5237:2:11 | m5237:3:12 | m5237:3:13 | m5237:3:14 | m5237:3:15 | ++-----------------------------------+-----------+-----------+-----------+-----------+-----------+-----------+-----------+-----------+-----------+-----------+------------+------------+------------+------------+------------+------------+ +| Runtime (RDTSC) [s] | 306.1496 | 306.1496 | 306.1496 | 306.1496 | 306.1241 | 306.1241 | 306.1241 | 306.1241 | 306.1456 | 306.1456 | 306.1456 | 306.1456 | 306.1473 | 306.1473 | 306.1473 | 306.1473 | +| Runtime unhalted [s] | 309.5251 | 299.6855 | 300.6908 | 303.5383 | 295.7213 | 286.2632 | 294.1686 | 293.5959 | 302.7623 | 291.6156 | 298.3640 | 298.3391 | 297.4647 | 287.9437 | 295.2859 | 293.9120 | +| Clock [MHz] | 3002.8752 | 3000.9393 | 3001.7712 | 3002.3784 | 3003.4387 | 3000.9142 | 3001.6559 | 3002.2404 | 3005.9542 | 3002.3871 | 3004.9692 | 3005.4602 | 3005.3877 | 3002.1022 | 3005.0518 | 3005.7968 | +| CPI | 0.8607 | 0.8487 | 0.8264 | 0.8238 | 0.8573 | 0.8508 | 0.8275 | 0.8229 | 0.8584 | 0.8515 | 0.8307 | 0.8225 | 0.8563 | 0.8472 | 0.8271 | 0.8185 | +| Memory read bandwidth [MBytes/s] | 1057.3707 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1026.1885 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| Memory read data volume [GBytes] | 323.7136 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 314.1631 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| Memory write bandwidth [MBytes/s] | 457.3470 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 433.3546 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| Memory write data volume [GBytes] | 140.0166 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 132.6696 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| Memory bandwidth [MBytes/s] | 1514.7176 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1459.5430 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| Memory data volume [GBytes] | 463.7302 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 446.8327 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | ++-----------------------------------+-----------+-----------+-----------+-----------+-----------+-----------+-----------+-----------+-----------+-----------+------------+------------+------------+------------+------------+------------+ + ++----------------------------------------+------------+-----------+-----------+-----------+-----------+-----------+-----------+ +| Metric | Sum | Min | Max | Avg | %ile 25 | %ile 50 | %ile 75 | ++----------------------------------------+------------+-----------+-----------+-----------+-----------+-----------+-----------+ +| Runtime (RDTSC) [s] STAT | 4898.2664 | 306.1241 | 306.1496 | 306.1416 | 306.1241 | 306.1456 | 306.1473 | +| Runtime unhalted [s] STAT | 4748.8760 | 286.2632 | 309.5251 | 296.8048 | 293.5959 | 295.7213 | 299.6855 | +| Clock [MHz] STAT | 48053.3225 | 3000.9142 | 3005.9542 | 3003.3327 | 3001.7712 | 3002.3871 | 3005.0518 | +| CPI STAT | 13.4303 | 0.8185 | 0.8607 | 0.8394 | 0.8238 | 0.8307 | 0.8515 | +| Memory read bandwidth [MBytes/s] STAT | 2083.5592 | 0 | 1057.3707 | 130.2224 | 0 | 0 | 0 | +| Memory read data volume [GBytes] STAT | 637.8767 | 0 | 323.7136 | 39.8673 | 0 | 0 | 0 | +| Memory write bandwidth [MBytes/s] STAT | 890.7016 | 0 | 457.3470 | 55.6688 | 0 | 0 | 0 | +| Memory write data volume [GBytes] STAT | 272.6862 | 0 | 140.0166 | 17.0429 | 0 | 0 | 0 | +| Memory bandwidth [MBytes/s] STAT | 2974.2606 | 0 | 1514.7176 | 185.8913 | 0 | 0 | 0 | +| Memory data volume [GBytes] STAT | 910.5629 | 0 | 463.7302 | 56.9102 | 0 | 0 | 0 | ++----------------------------------------+------------+-----------+-----------+-----------+-----------+-----------+-----------+ +``` + +#### Floating-point performance (`--enable-debug`) + +##### 100 x 100 humans on 10 x 10 km for minimum(10000 steps, 10 minutes): + +``` +likwid-mpirun -np 4 -t 4 -omp intel -g FLOPS_DP -- swift_mpi --threads=4 -A -s -g -G --hm-river --hm-randomwalk -n 10000 ${HUMANMOBILITY}.yml +``` +``` +Group: 1 ++--------------------------------------+---------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+-------------+--------------+--------------+--------------+-------------+--------------+--------------+ +| Event | Counter | m5335:0:0 | m5335:0:1 | m5335:0:2 | m5335:0:3 | m5335:1:4 | m5335:1:5 | m5335:1:6 | m5335:1:7 | m5335:2:8 | m5335:2:9 | m5335:2:10 | m5335:2:11 | m5335:3:12 | m5335:3:13 | m5335:3:14 | m5335:3:15 | ++--------------------------------------+---------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+-------------+--------------+--------------+--------------+-------------+--------------+--------------+ +| INSTR_RETIRED_ANY | FIXC0 | 321639846446 | 95295603041 | 108910848024 | 155472825044 | 313546724897 | 94256234895 | 108124055942 | 153092993255 | 354102823655 | 90172462296 | 103862094786 | 147648439350 | 348594216141 | 89095844083 | 102221345209 | 147222018909 | +| CPU_CLK_UNHALTED_CORE | FIXC1 | 270939072666 | 108795059797 | 73441745166 | 136871060681 | 270188518834 | 106475416429 | 74103623128 | 136847769255 | 279981732694 | 99721523895 | 71746250642 | 129586517363 | 279131669583 | 98854040792 | 70657886881 | 128905664172 | +| CPU_CLK_UNHALTED_REF | FIXC2 | 226196112298 | 93182929476 | 63301619290 | 117267745842 | 225514052218 | 91209295294 | 63836360484 | 117253940180 | 231285837224 | 85652307312 | 61703251714 | 111148431316 | 230557302768 | 84904931202 | 60771101846 | 110549719124 | +| FP_COMP_OPS_EXE_SSE_FP_PACKED_DOUBLE | PMC0 | 1233656017 | 299866399 | 207790973 | 267129364 | 1436475677 | 283846907 | 192877546 | 260322176 | 849148674 | 293379462 | 203124677 | 267482063 | 754057509 | 286105969 | 196003709 | 258701926 | +| FP_COMP_OPS_EXE_SSE_FP_SCALAR_DOUBLE | PMC1 | 4928924598 | 1450613650 | 605275577 | 1301031710 | 5716699045 | 1423932221 | 568589565 | 1313471659 | 3411173582 | 1341345940 | 586246720 | 1242421796 | 3063556224 | 1343623328 | 567353486 | 1228806835 | +| SIMD_FP_256_PACKED_DOUBLE | PMC2 | 210446767 | 197850878 | 113108074 | 140181433 | 196810155 | 187607559 | 104398736 | 134087686 | 205509225 | 192591120 | 107716587 | 140119426 | 197362833 | 186098897 | 102038631 | 132977922 | ++--------------------------------------+---------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+-------------+--------------+--------------+--------------+-------------+--------------+--------------+ + ++-------------------------------------------+---------+---------------+-------------+--------------+--------------+ +| Event | Counter | Sum | Min | Max | Avg | ++-------------------------------------------+---------+---------------+-------------+--------------+--------------+ +| INSTR_RETIRED_ANY STAT | FIXC0 | 2733258375973 | 89095844083 | 354102823655 | 1.708286e+11 | +| CPU_CLK_UNHALTED_CORE STAT | FIXC1 | 2336247551978 | 70657886881 | 279981732694 | 1.460155e+11 | +| CPU_CLK_UNHALTED_REF STAT | FIXC2 | 1974334937588 | 60771101846 | 231285837224 | 1.233959e+11 | +| FP_COMP_OPS_EXE_SSE_FP_PACKED_DOUBLE STAT | PMC0 | 7289969048 | 192877546 | 1436475677 | 4.556231e+08 | +| FP_COMP_OPS_EXE_SSE_FP_SCALAR_DOUBLE STAT | PMC1 | 30093065936 | 567353486 | 5716699045 | 1880816621 | +| SIMD_FP_256_PACKED_DOUBLE STAT | PMC2 | 2548905929 | 102038631 | 210446767 | 1.593066e+08 | ++-------------------------------------------+---------+---------------+-------------+--------------+--------------+ + ++-------------------------+-----------+-----------+-----------+-----------+-----------+-----------+-----------+-----------+-----------+-----------+------------+------------+------------+------------+------------+------------+ +| Metric | m5335:0:0 | m5335:0:1 | m5335:0:2 | m5335:0:3 | m5335:1:4 | m5335:1:5 | m5335:1:6 | m5335:1:7 | m5335:2:8 | m5335:2:9 | m5335:2:10 | m5335:2:11 | m5335:3:12 | m5335:3:13 | m5335:3:14 | m5335:3:15 | ++-------------------------+-----------+-----------+-----------+-----------+-----------+-----------+-----------+-----------+-----------+-----------+------------+------------+------------+------------+------------+------------+ +| Runtime (RDTSC) [s] | 132.3468 | 132.3468 | 132.3468 | 132.3468 | 132.3353 | 132.3353 | 132.3353 | 132.3353 | 132.3355 | 132.3355 | 132.3355 | 132.3355 | 132.3330 | 132.3330 | 132.3330 | 132.3330 | +| Runtime unhalted [s] | 104.2074 | 41.8443 | 28.2469 | 52.6428 | 103.9211 | 40.9530 | 28.5021 | 52.6350 | 107.6876 | 38.3553 | 27.5953 | 49.8421 | 107.3587 | 38.0209 | 27.1762 | 49.5793 | +| Clock [MHz] | 3114.2927 | 3035.6084 | 3016.4845 | 3034.6317 | 3114.9884 | 3035.1028 | 3018.1059 | 3034.4042 | 3147.3464 | 3027.0091 | 3023.1176 | 3031.2403 | 3147.7637 | 3027.1462 | 3022.9810 | 3031.7001 | +| CPI | 0.8424 | 1.1417 | 0.6743 | 0.8804 | 0.8617 | 1.1296 | 0.6854 | 0.8939 | 0.7907 | 1.1059 | 0.6908 | 0.8777 | 0.8007 | 1.1095 | 0.6912 | 0.8756 | +| DP [MFLOP/s] | 62.2457 | 21.4720 | 11.1320 | 18.1041 | 70.8571 | 20.7205 | 10.3671 | 17.9126 | 44.8217 | 20.3911 | 10.7557 | 17.6662 | 40.5124 | 20.1026 | 10.3339 | 17.2151 | +| AVX DP [MFLOP/s] | 6.3605 | 5.9798 | 3.4185 | 4.2368 | 5.9488 | 5.6707 | 3.1556 | 4.0530 | 6.2118 | 5.8213 | 3.2559 | 4.2353 | 5.9656 | 5.6252 | 3.0843 | 4.0195 | +| Packed [MUOPS/s] | 10.9115 | 3.7607 | 2.4247 | 3.0776 | 12.3420 | 3.5626 | 2.2464 | 2.9804 | 7.9696 | 3.6723 | 2.3489 | 3.0801 | 7.1896 | 3.5683 | 2.2522 | 2.9598 | +| Scalar [MUOPS/s] | 37.2425 | 10.9607 | 4.5734 | 9.8305 | 43.1986 | 10.7600 | 4.2966 | 9.9253 | 25.7767 | 10.1359 | 4.4300 | 9.3884 | 23.1504 | 10.1534 | 4.2873 | 9.2857 | +| Vectorization ratio [%] | 22.6596 | 25.5458 | 34.6478 | 23.8425 | 22.2216 | 24.8738 | 34.3328 | 23.0935 | 23.6162 | 26.5948 | 34.6500 | 24.7028 | 23.6968 | 26.0049 | 34.4400 | 24.1705 | ++-------------------------+-----------+-----------+-----------+-----------+-----------+-----------+-----------+-----------+-----------+-----------+------------+------------+------------+------------+------------+------------+ + ++------------------------------+------------+-----------+-----------+-----------+-----------+-----------+-----------+ +| Metric | Sum | Min | Max | Avg | %ile 25 | %ile 50 | %ile 75 | ++------------------------------+------------+-----------+-----------+-----------+-----------+-----------+-----------+ +| Runtime (RDTSC) [s] STAT | 2117.4024 | 132.3330 | 132.3468 | 132.3376 | 132.3330 | 132.3353 | 132.3355 | +| Runtime unhalted [s] STAT | 898.5680 | 27.1762 | 107.6876 | 56.1605 | 107.6876 | 28.5021 | 41.8443 | +| Clock [MHz] STAT | 48861.9230 | 3016.4845 | 3147.7637 | 3053.8702 | 3023.1176 | 3031.7001 | 3035.6084 | +| CPI STAT | 14.0515 | 0.6743 | 1.1417 | 0.8782 | 0.6912 | 0.8617 | 0.8939 | +| DP [MFLOP/s] STAT | 414.6098 | 10.3339 | 70.8571 | 25.9131 | 11.1320 | 18.1041 | 21.4720 | +| AVX DP [MFLOP/s] STAT | 77.0426 | 3.0843 | 6.3605 | 4.8152 | 3.4185 | 4.2368 | 5.9488 | +| Packed [MUOPS/s] STAT | 74.3467 | 2.2464 | 12.3420 | 4.6467 | 2.2522 | 2.9804 | 3.5683 | +| Scalar [MUOPS/s] STAT | 227.3954 | 4.2873 | 43.1986 | 14.2122 | 10.9607 | 4.2873 | 43.1986 | +| Vectorization ratio [%] STAT | 429.0934 | 22.2216 | 34.6500 | 26.8183 | 23.6162 | 24.7028 | 26.5948 | ++------------------------------+------------+-----------+-----------+-----------+-----------+-----------+-----------+ +``` + +##### 1000 x 1000 humans on 100 x 100 km for minimum(1000 steps, 10 minutes): + +``` +likwid-mpirun -np 4 -t 4 -omp intel -g FLOPS_DP -- swift_mpi --threads=4 -A -s -g -G --hm-river --hm-randomwalk -n 1000 ${HUMANMOBILITY}.yml +``` +``` +Group: 1 ++--------------------------------------+---------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+---------------+ +| Event | Counter | m5334:0:0 | m5334:0:1 | m5334:0:2 | m5334:0:3 | m5334:1:4 | m5334:1:5 | m5334:1:6 | m5334:1:7 | m5334:2:8 | m5334:2:9 | m5334:2:10 | m5334:2:11 | m5334:3:12 | m5334:3:13 | m5334:3:14 | m5334:3:15 | ++--------------------------------------+---------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+---------------+ +| INSTR_RETIRED_ANY | FIXC0 | 928603457884 | 911573592017 | 969853890424 | 975888043593 | 990161259348 | 966007358758 | 980630430015 | 997507812056 | 949733963252 | 928128991523 | 980345658060 | 996101066135 | 964132995603 | 940658959069 | 998496468941 | 1007029168610 | +| CPU_CLK_UNHALTED_CORE | FIXC1 | 793709132880 | 770955305231 | 799673078449 | 799040042510 | 841996850660 | 816201744613 | 807755583411 | 814456886578 | 810874685654 | 782474264230 | 808541053373 | 813750612603 | 823452300321 | 793851472176 | 812653860819 | 816543101779 | +| CPU_CLK_UNHALTED_REF | FIXC2 | 687190362924 | 667981690480 | 692733283216 | 692082389804 | 728841882236 | 707198316214 | 699755009798 | 705453081450 | 701540400028 | 677807241384 | 699089757886 | 703420064984 | 712007211786 | 687443295084 | 702251906564 | 705448866694 | +| FP_COMP_OPS_EXE_SSE_FP_PACKED_DOUBLE | PMC0 | 2446903175 | 2539497409 | 2595236120 | 3596634676 | 2543018613 | 2653218612 | 2881304512 | 3667504788 | 2426244399 | 2644511043 | 2551783983 | 3716724177 | 2502062051 | 2633380750 | 2635341656 | 3560837119 | +| FP_COMP_OPS_EXE_SSE_FP_SCALAR_DOUBLE | PMC1 | 3392789115 | 3292957442 | 3205400447 | 4053329366 | 3470664633 | 3477100065 | 3488209717 | 4221351979 | 3407043697 | 3406085381 | 3167849851 | 4203688028 | 3415853821 | 3450611329 | 3248905361 | 4075688959 | +| SIMD_FP_256_PACKED_DOUBLE | PMC2 | 3797006147 | 3891979016 | 3924927923 | 3314325646 | 4333749324 | 4184528274 | 4175025409 | 3810679152 | 4032276636 | 4071116005 | 4021960270 | 3476932297 | 4103566526 | 4129436531 | 4058977364 | 3650775699 | ++--------------------------------------+---------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+---------------+ + ++-------------------------------------------+---------+----------------+--------------+---------------+--------------+ +| Event | Counter | Sum | Min | Max | Avg | ++-------------------------------------------+---------+----------------+--------------+---------------+--------------+ +| INSTR_RETIRED_ANY STAT | FIXC0 | 15484853115288 | 911573592017 | 1007029168610 | 9.678033e+11 | +| CPU_CLK_UNHALTED_CORE STAT | FIXC1 | 12905929975287 | 770955305231 | 841996850660 | 8.066206e+11 | +| CPU_CLK_UNHALTED_REF STAT | FIXC2 | 11170244760532 | 667981690480 | 728841882236 | 6.981403e+11 | +| FP_COMP_OPS_EXE_SSE_FP_PACKED_DOUBLE STAT | PMC0 | 45594203083 | 2426244399 | 3716724177 | 2.849638e+09 | +| FP_COMP_OPS_EXE_SSE_FP_SCALAR_DOUBLE STAT | PMC1 | 56977529191 | 3167849851 | 4221351979 | 3.561096e+09 | +| SIMD_FP_256_PACKED_DOUBLE STAT | PMC2 | 62977262219 | 3314325646 | 4333749324 | 3.936079e+09 | ++-------------------------------------------+---------+----------------+--------------+---------------+--------------+ + ++-------------------------+-----------+-----------+-----------+-----------+-----------+-----------+-----------+-----------+-----------+-----------+------------+------------+------------+------------+------------+------------+ +| Metric | m5334:0:0 | m5334:0:1 | m5334:0:2 | m5334:0:3 | m5334:1:4 | m5334:1:5 | m5334:1:6 | m5334:1:7 | m5334:2:8 | m5334:2:9 | m5334:2:10 | m5334:2:11 | m5334:3:12 | m5334:3:13 | m5334:3:14 | m5334:3:15 | ++-------------------------+-----------+-----------+-----------+-----------+-----------+-----------+-----------+-----------+-----------+-----------+------------+------------+------------+------------+------------+------------+ +| Runtime (RDTSC) [s] | 323.8998 | 323.8998 | 323.8998 | 323.8998 | 323.9200 | 323.9200 | 323.9200 | 323.9200 | 323.9356 | 323.9356 | 323.9356 | 323.9356 | 323.9037 | 323.9037 | 323.9037 | 323.9037 | +| Runtime unhalted [s] | 305.2741 | 296.5226 | 307.5680 | 307.3245 | 323.8471 | 313.9259 | 310.6773 | 313.2548 | 311.8845 | 300.9610 | 310.9870 | 312.9907 | 316.7138 | 305.3289 | 312.5606 | 314.0564 | +| Clock [MHz] | 3003.0024 | 3000.7928 | 3001.3579 | 3001.8024 | 3003.6377 | 3000.7285 | 3001.2650 | 3001.7213 | 3005.1141 | 3001.3992 | 3006.9696 | 3007.7136 | 3006.9449 | 3002.4362 | 3008.7360 | 3009.4351 | +| CPI | 0.8547 | 0.8457 | 0.8245 | 0.8188 | 0.8504 | 0.8449 | 0.8237 | 0.8165 | 0.8538 | 0.8431 | 0.8248 | 0.8169 | 0.8541 | 0.8439 | 0.8139 | 0.8108 | +| DP [MFLOP/s] | 72.4749 | 73.9113 | 74.3921 | 75.6527 | 79.9324 | 78.7900 | 80.1152 | 82.7336 | 75.2885 | 77.1128 | 75.1978 | 78.8579 | 76.6717 | 77.9093 | 76.4286 | 79.6547 | +| AVX DP [MFLOP/s] | 46.8911 | 48.0640 | 48.4709 | 40.9303 | 53.5163 | 51.6736 | 51.5563 | 47.0570 | 49.7911 | 50.2707 | 49.6637 | 42.9336 | 50.6764 | 50.9959 | 50.1257 | 45.0847 | +| Packed [MUOPS/s] | 19.2773 | 19.8564 | 20.1302 | 21.3367 | 21.2298 | 21.1094 | 21.7842 | 23.0865 | 19.9377 | 20.7314 | 20.2934 | 22.2071 | 20.3938 | 20.8791 | 20.6676 | 22.2647 | +| Scalar [MUOPS/s] | 10.4748 | 10.1666 | 9.8963 | 12.5141 | 10.7146 | 10.7344 | 10.7687 | 13.0321 | 10.5177 | 10.5147 | 9.7793 | 12.9769 | 10.5459 | 10.6532 | 10.0305 | 12.5830 | +| Vectorization ratio [%] | 64.7930 | 66.1373 | 67.0415 | 63.0315 | 66.4587 | 66.2903 | 66.9193 | 63.9186 | 65.4653 | 66.3487 | 67.4812 | 63.1170 | 65.9147 | 66.2150 | 67.3254 | 63.8914 | ++-------------------------+-----------+-----------+-----------+-----------+-----------+-----------+-----------+-----------+-----------+-----------+------------+------------+------------+------------+------------+------------+ + ++------------------------------+------------+-----------+-----------+-----------+-----------+-----------+-----------+ +| Metric | Sum | Min | Max | Avg | %ile 25 | %ile 50 | %ile 75 | ++------------------------------+------------+-----------+-----------+-----------+-----------+-----------+-----------+ +| Runtime (RDTSC) [s] STAT | 5182.6364 | 323.8998 | 323.9356 | 323.9148 | 323.8998 | 323.9037 | 323.9200 | +| Runtime unhalted [s] STAT | 4963.8772 | 296.5226 | 323.8471 | 310.2423 | 305.3289 | 310.9870 | 313.2548 | +| Clock [MHz] STAT | 48063.0567 | 3000.7285 | 3009.4351 | 3003.9410 | 3001.3579 | 3002.4362 | 3006.9449 | +| CPI STAT | 13.3405 | 0.8108 | 0.8547 | 0.8338 | 0.8169 | 0.8248 | 0.8457 | +| DP [MFLOP/s] STAT | 1235.1235 | 72.4749 | 82.7336 | 77.1952 | 75.1978 | 76.6717 | 78.8579 | +| AVX DP [MFLOP/s] STAT | 777.7013 | 40.9303 | 53.5163 | 48.6063 | 46.8911 | 49.6637 | 50.6764 | +| Packed [MUOPS/s] STAT | 335.1853 | 19.2773 | 23.0865 | 20.9491 | 20.1302 | 20.7314 | 21.3367 | +| Scalar [MUOPS/s] STAT | 175.9028 | 9.7793 | 13.0321 | 10.9939 | 10.5147 | 10.7146 | 12.5830 | +| Vectorization ratio [%] STAT | 1050.3489 | 63.0315 | 67.4812 | 65.6468 | 63.9186 | 66.1373 | 66.4587 | ++------------------------------+------------+-----------+-----------+-----------+-----------+-----------+-----------+ +``` + +#### Comparison wrt scaling and with microbenchmarks + +--- + +1. **Scaling**: 100x100 Humans (10km x 10km) vs 1000x1000 Humans (100km x 100km) + +**Memory Metrics** (`MEM` group, best rank) + +| Metric | 100x100 Humans | 1000x1000 Humans | Change | +|-------------------------------|----------------|------------------|-----------------------| +| Runtime (RDTSC) [s] | 136.69 | 306.15 | ~2.24x increase | +| Memory BW [MB/s] | 1427 | 1515 | Slight increase | +| Memory Data Volume [GB] | 195 | 464 | ~2.38x increase | +| CPI | 0.82 | 0.84 | Similar | + +**Floating-Point Metrics** (`FLOPS_DP` group, best rank) + +| Metric | 100x100 Humans | 1000x1000 Humans | Change | +|------------------|----------------|------------------|-----------------------| +| DP MFLOP/s | ~62 | ~44 | ~29% decrease | +| Vectorization [%]| ~22–34 | ~22–34 | Similar (low) | +| CPI | ~0.84 | ~0.84 | Similar | + +--- + +2. **Comparison with `likwid-bench triad`** + +| Metric | swift_mpi (best rank) | likwid-bench triad | Ratio | +|----------------|----------------------|--------------------|------------| +| Memory BW | 1515 MB/s | 26614.00 MB/s | ~5.7% | +| Read/Write Ratio | ~1.8:1 | 3:1 | Lower | +| CPI | 0.84 | 3.13 | ~3.7x better | + +- **Observation:** + - SWIFT achieves only ~5.7% of the streaming memory bandwidth of `triad`. + - Access pattern is less streaming (lower read/write ratio). + - CPI is better in SWIFT, but this is expected for memory-bound code. + +--- + +3. **Comparison with `likwid-bench peakflops`** + +| Metric | swift_mpi (best rank) | likwid-bench peakflops | Ratio | +|----------------|----------------------|------------------------|------------| +| MFLOP/s | ~82.7 | 4936.19 | ~1.7% | +| Memory BW | 1515 MB/s | 2468.09 MB/s | ~61% | +| CPI | 0.84 | 8.43 | ~10x better | +| Vectorization | ~67% | ~100% | ~0.67x | + +- **Observation:** + - SWIFT achieves about 1.7% of peak FLOPS and about 61% of `peakflops` memory bandwidth. + - Vectorization ratio is much lower than `peakflops`. + - CPI is much better in SWIFT, but `peakflops` is compute-bound. + +--- + +**Summary Table** + +| Metric | swift_mpi (large, best rank) | triad | peakflops | +|-----------------------|------------------------------|-------------------|-------------------| +| Memory BW [MB/s] | 1515 | 26614.00 | 2468.09 | +| DP MFLOP/s | ~82.7 | (not measured) | 4936.19 | +| CPI | 0.84 | 3.13 | 8.43 | +| Vectorization [%] | ~67 | (not measured) | ~100 | + +--- + +**Key Takeaways** + +- **Scaling:** Memory bandwidth and data volume increase with problem size, but bandwidth is still far from hardware peak. +- **Compared to `triad`:** SWIFT achieves only a small fraction of the streaming bandwidth, with a more balanced (less streaming) access pattern and better CPI. +- **Compared to `peakflops`:** SWIFT achieves about 1.7% of peak FLOPS, and about 61% of `peakflops` memory bandwidth, with much lower vectorization. +- **Optimization potential:** There is significant headroom for improving both memory access patterns and vectorization in SWIFT. + +#### Memory performance (`--enable-ipo`) + +##### 1000 x 1000 humans on 100 x 100 km for minimum, multiple rivers (100 steps, 10 minutes): +``` +likwid-perfctr -f -C 0-15 -g MEM swift -A -s -g -G --hm-river --hm-randomwalk --threads=16 -n 1000 humanMobility.yml +``` +``` +-------------------------------------------------------------------------------- +Group 1: MEM ++-----------------------+---------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+---------------+--------------+--------------+--------------+--------------+--------------+---------------+--------------+ +| Event | Counter | HWThread 0 | HWThread 1 | HWThread 2 | HWThread 3 | HWThread 4 | HWThread 5 | HWThread 6 | HWThread 7 | HWThread 8 | HWThread 9 | HWThread 10 | HWThread 11 | HWThread 12 | HWThread 13 | HWThread 14 | HWThread 15 | ++-----------------------+---------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+---------------+--------------+--------------+--------------+--------------+--------------+---------------+--------------+ +| INSTR_RETIRED_ANY | FIXC0 | 975876943752 | 965752585140 | 993753825451 | 979073598842 | 970379226841 | 978056530864 | 963295174486 | 978504614942 | 1025397969559 | 999298716967 | 993478925405 | 998344253860 | 974925642167 | 986585240716 | 1008142361518 | 958684471716 | +| CPU_CLK_UNHALTED_CORE | FIXC1 | 887566984243 | 875916485053 | 900089754704 | 886926244471 | 879167838974 | 884625668090 | 871514404512 | 880467057248 | 922567316942 | 902274334852 | 897231000281 | 899206842936 | 881539711541 | 890218795683 | 905108770926 | 849800952747 | +| CPU_CLK_UNHALTED_REF | FIXC2 | 768140274512 | 759043739714 | 779977106012 | 768588381990 | 761894173040 | 766590485856 | 755228198478 | 762985594280 | 799331217282 | 781861953288 | 777521797962 | 779256954554 | 763927306012 | 771449474224 | 784324764002 | 736409433578 | +| CAS_COUNT_RD | MBOX0C0 | 4261444731 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3504353741 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| CAS_COUNT_WR | MBOX0C1 | 1512708396 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 943101388 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| CAS_COUNT_RD | MBOX1C0 | 4726480585 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 4403350275 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| CAS_COUNT_WR | MBOX1C1 | 1539877534 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1415800887 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| CAS_COUNT_RD | MBOX2C0 | 9218646904 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 5702862507 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| CAS_COUNT_WR | MBOX2C1 | 4397646832 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1439969824 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| CAS_COUNT_RD | MBOX3C0 | 10524023451 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 8511873950 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| CAS_COUNT_WR | MBOX3C1 | 2136606278 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 3398124012 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | ++-----------------------+---------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+--------------+---------------+--------------+--------------+--------------+--------------+--------------+---------------+--------------+ + ++----------------------------+---------+----------------+--------------+---------------+--------------+ +| Event | Counter | Sum | Min | Max | Avg | ++----------------------------+---------+----------------+--------------+---------------+--------------+ +| INSTR_RETIRED_ANY STAT | FIXC0 | 15749550082226 | 958684471716 | 1025397969559 | 9.843469e+11 | +| CPU_CLK_UNHALTED_CORE STAT | FIXC1 | 14214222163203 | 849800952747 | 922567316942 | 8.883889e+11 | +| CPU_CLK_UNHALTED_REF STAT | FIXC2 | 12316530854784 | 736409433578 | 799331217282 | 769783178424 | +| CAS_COUNT_RD STAT | MBOX0C0 | 7765798472 | 0 | 4261444731 | 4.853624e+08 | +| CAS_COUNT_WR STAT | MBOX0C1 | 2455809784 | 0 | 1512708396 | 1.534881e+08 | +| CAS_COUNT_RD STAT | MBOX1C0 | 9129830860 | 0 | 4726480585 | 5.706144e+08 | +| CAS_COUNT_WR STAT | MBOX1C1 | 2955678421 | 0 | 1539877534 | 1.847299e+08 | +| CAS_COUNT_RD STAT | MBOX2C0 | 14921509411 | 0 | 9218646904 | 9.325943e+08 | +| CAS_COUNT_WR STAT | MBOX2C1 | 5837616656 | 0 | 4397646832 | 364851041 | +| CAS_COUNT_RD STAT | MBOX3C0 | 19035897401 | 0 | 10524023451 | 1.189744e+09 | +| CAS_COUNT_WR STAT | MBOX3C1 | 5534730290 | 0 | 3398124012 | 3.459206e+08 | ++----------------------------+---------+----------------+--------------+---------------+--------------+ + ++-----------------------------------+------------+------------+------------+------------+------------+------------+------------+------------+------------+------------+-------------+-------------+-------------+-------------+-------------+-------------+ +| Metric | HWThread 0 | HWThread 1 | HWThread 2 | HWThread 3 | HWThread 4 | HWThread 5 | HWThread 6 | HWThread 7 | HWThread 8 | HWThread 9 | HWThread 10 | HWThread 11 | HWThread 12 | HWThread 13 | HWThread 14 | HWThread 15 | ++-----------------------------------+------------+------------+------------+------------+------------+------------+------------+------------+------------+------------+-------------+-------------+-------------+-------------+-------------+-------------+ +| Runtime (RDTSC) [s] | 1025.7361 | 1025.7361 | 1025.7361 | 1025.7361 | 1025.7361 | 1025.7361 | 1025.7361 | 1025.7361 | 1025.7361 | 1025.7361 | 1025.7361 | 1025.7361 | 1025.7361 | 1025.7361 | 1025.7361 | 1025.7361 | +| Runtime unhalted [s] | 341.3757 | 336.8947 | 346.1922 | 341.1293 | 338.1452 | 340.2444 | 335.2016 | 338.6449 | 354.8375 | 347.0324 | 345.0927 | 345.8526 | 339.0575 | 342.3956 | 348.1226 | 326.8501 | +| Clock [MHz] | 3004.2020 | 3000.2982 | 3000.3539 | 3000.2830 | 3000.1686 | 3000.2997 | 3000.3015 | 3000.3043 | 3000.8191 | 3000.3854 | 3000.2692 | 3000.1809 | 3000.2564 | 3000.2524 | 3000.3600 | 3000.3118 | +| CPI | 0.9095 | 0.9070 | 0.9057 | 0.9059 | 0.9060 | 0.9045 | 0.9047 | 0.8998 | 0.8997 | 0.9029 | 0.9031 | 0.9007 | 0.9042 | 0.9023 | 0.8978 | 0.8864 | +| Memory read bandwidth [MBytes/s] | 1792.6230 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1380.3123 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| Memory read data volume [GBytes] | 1838.7581 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1415.8362 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| Memory write bandwidth [MBytes/s] | 598.1633 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 449.0509 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| Memory write data volume [GBytes] | 613.5577 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 460.6078 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| Memory bandwidth [MBytes/s] | 2390.7863 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1829.3633 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | +| Memory data volume [GBytes] | 2452.3158 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | 1876.4439 | 0 | 0 | 0 | 0 | 0 | 0 | 0 | ++-----------------------------------+------------+------------+------------+------------+------------+------------+------------+------------+------------+------------+-------------+-------------+-------------+-------------+-------------+-------------+ + ++----------------------------------------+------------+-----------+-----------+-----------+ +| Metric | Sum | Min | Max | Avg | ++----------------------------------------+------------+-----------+-----------+-----------+ +| Runtime (RDTSC) [s] STAT | 16411.7776 | 1025.7361 | 1025.7361 | 1025.7361 | +| Runtime unhalted [s] STAT | 5467.0690 | 326.8501 | 354.8375 | 341.6918 | +| Clock [MHz] STAT | 48009.0464 | 3000.1686 | 3004.2020 | 3000.5654 | +| CPI STAT | 14.4402 | 0.8864 | 0.9095 | 0.9025 | +| Memory read bandwidth [MBytes/s] STAT | 3172.9353 | 0 | 1792.6230 | 198.3085 | +| Memory read data volume [GBytes] STAT | 3254.5943 | 0 | 1838.7581 | 203.4121 | +| Memory write bandwidth [MBytes/s] STAT | 1047.2142 | 0 | 598.1633 | 65.4509 | +| Memory write data volume [GBytes] STAT | 1074.1655 | 0 | 613.5577 | 67.1353 | +| Memory bandwidth [MBytes/s] STAT | 4220.1496 | 0 | 2390.7863 | 263.7594 | +| Memory data volume [GBytes] STAT | 4328.7597 | 0 | 2452.3158 | 270.5475 | ++----------------------------------------+------------+-----------+-----------+-----------+ +``` + +#### Floating-point performance (`--enable-ipo`) + +##### 1000 x 1000 humans on 100 x 100 km for minimum, multiple rivers (1000 steps, 10 minutes): +``` +likwid-perfctr -f -C 0 -g FLOPS_DP swift -A -s -g -G --hm-river --hm-randomwalk --threads=16 -n 100 humanMobility.yml +``` +``` +-------------------------------------------------------------------------------- +Group 1: FLOPS_DP ++--------------------------------------+---------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+ +| Event | Counter | HWThread 0 | HWThread 1 | HWThread 2 | HWThread 3 | HWThread 4 | HWThread 5 | HWThread 6 | HWThread 7 | HWThread 8 | HWThread 9 | HWThread 10 | HWThread 11 | HWThread 12 | HWThread 13 | HWThread 14 | HWThread 15 | ++--------------------------------------+---------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+ +| INSTR_RETIRED_ANY | FIXC0 | 1062384074352 | 1036415071743 | 1052600966172 | 1054789692902 | 1067560654541 | 1047140772969 | 1063377721926 | 1070395853218 | 1039607704349 | 1039203944144 | 1021167287992 | 1053327830351 | 1029103907737 | 1065359127658 | 1035985100198 | 1058987431644 | +| CPU_CLK_UNHALTED_CORE | FIXC1 | 962368120170 | 937971084671 | 951319896126 | 952615880874 | 962674274203 | 942721837444 | 958001948413 | 963713542269 | 941090357070 | 937104057117 | 924319305694 | 949240686047 | 928964172167 | 959757595020 | 933028997627 | 936503975787 | +| CPU_CLK_UNHALTED_REF | FIXC2 | 832954836298 | 812828889132 | 824416687198 | 825565709150 | 834282085390 | 816969631036 | 830207617928 | 835157406044 | 815491942954 | 812137083732 | 801021984958 | 822586472682 | 805038710476 | 831556644412 | 808526438616 | 811535268284 | +| FP_COMP_OPS_EXE_SSE_FP_PACKED_DOUBLE | PMC0 | 3590712378 | 3346231008 | 3406454917 | 3453499476 | 3557858528 | 3403706927 | 3497341900 | 3419434893 | 3557593240 | 3426335665 | 3473419347 | 3509474601 | 3294336813 | 3550051922 | 3434509625 | 3434216267 | +| FP_COMP_OPS_EXE_SSE_FP_SCALAR_DOUBLE | PMC1 | 4898009353 | 4427932292 | 4503997056 | 4539411167 | 4682085388 | 4502844923 | 4601153705 | 4517165183 | 4576502676 | 4513198827 | 4522478313 | 4609811691 | 4314254144 | 4662173507 | 4508650566 | 4232780054 | +| SIMD_FP_256_PACKED_DOUBLE | PMC2 | 5395188163 | 5189979707 | 5280222115 | 5175091452 | 5270021723 | 5318743192 | 5209055517 | 5231177899 | 5066647603 | 5242994284 | 5184493447 | 5258492332 | 5032011937 | 5335969911 | 5365858948 | 5006897884 | ++--------------------------------------+---------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+---------------+ + ++-------------------------------------------+---------+----------------+---------------+---------------+--------------+ +| Event | Counter | Sum | Min | Max | Avg | ++-------------------------------------------+---------+----------------+---------------+---------------+--------------+ +| INSTR_RETIRED_ANY STAT | FIXC0 | 16797407141896 | 1021167287992 | 1070395853218 | 1.049838e+12 | +| CPU_CLK_UNHALTED_CORE STAT | FIXC1 | 15141395730699 | 924319305694 | 963713542269 | 9.463372e+11 | +| CPU_CLK_UNHALTED_REF STAT | FIXC2 | 13120277408290 | 801021984958 | 835157406044 | 8.200173e+11 | +| FP_COMP_OPS_EXE_SSE_FP_PACKED_DOUBLE STAT | PMC0 | 55355177507 | 3294336813 | 3590712378 | 3.459699e+09 | +| FP_COMP_OPS_EXE_SSE_FP_SCALAR_DOUBLE STAT | PMC1 | 72612448845 | 4232780054 | 4898009353 | 4.538278e+09 | +| SIMD_FP_256_PACKED_DOUBLE STAT | PMC2 | 83562846114 | 5006897884 | 5395188163 | 5.222678e+09 | ++-------------------------------------------+---------+----------------+---------------+---------------+--------------+ + ++-------------------------+------------+------------+------------+------------+------------+------------+------------+------------+------------+------------+-------------+-------------+-------------+-------------+-------------+-------------+ +| Metric | HWThread 0 | HWThread 1 | HWThread 2 | HWThread 3 | HWThread 4 | HWThread 5 | HWThread 6 | HWThread 7 | HWThread 8 | HWThread 9 | HWThread 10 | HWThread 11 | HWThread 12 | HWThread 13 | HWThread 14 | HWThread 15 | ++-------------------------+------------+------------+------------+------------+------------+------------+------------+------------+------------+------------+-------------+-------------+-------------+-------------+-------------+-------------+ +| Runtime (RDTSC) [s] | 1115.6544 | 1115.6544 | 1115.6544 | 1115.6544 | 1115.6544 | 1115.6544 | 1115.6544 | 1115.6544 | 1115.6544 | 1115.6544 | 1115.6544 | 1115.6544 | 1115.6544 | 1115.6544 | 1115.6544 | 1115.6544 | +| Runtime unhalted [s] | 370.1459 | 360.7623 | 365.8965 | 366.3950 | 370.2637 | 362.5896 | 368.4666 | 370.6634 | 361.9621 | 360.4288 | 355.5116 | 365.0968 | 357.2981 | 369.1418 | 358.8615 | 360.1980 | +| Clock [MHz] | 3003.9178 | 3000.2580 | 3000.1853 | 3000.0911 | 3000.0930 | 3000.1705 | 3000.1844 | 3000.1841 | 3000.4053 | 3000.0380 | 3000.1700 | 3000.2888 | 3000.2019 | 3000.8065 | 3000.3312 | 3000.3402 | +| CPI | 0.9059 | 0.9050 | 0.9038 | 0.9031 | 0.9018 | 0.9003 | 0.9009 | 0.9003 | 0.9052 | 0.9018 | 0.9052 | 0.9012 | 0.9027 | 0.9009 | 0.9006 | 0.8843 | +| DP [MFLOP/s] | 30.1708 | 28.5754 | 29.0751 | 28.8143 | 29.4696 | 29.2073 | 29.0700 | 28.9344 | 28.6453 | 28.9855 | 28.8685 | 29.2767 | 27.8141 | 29.6742 | 29.4366 | 27.9018 | +| AVX DP [MFLOP/s] | 19.3436 | 18.6078 | 18.9314 | 18.5545 | 18.8948 | 19.0695 | 18.6762 | 18.7555 | 18.1657 | 18.7979 | 18.5882 | 18.8535 | 18.0415 | 19.1313 | 19.2384 | 17.9514 | +| Packed [MUOPS/s] | 8.0544 | 7.6513 | 7.7862 | 7.7341 | 7.9127 | 7.8182 | 7.8038 | 7.7538 | 7.7302 | 7.7706 | 7.7604 | 7.8590 | 7.4632 | 7.9649 | 7.8881 | 7.5661 | +| Scalar [MUOPS/s] | 4.3903 | 3.9689 | 4.0371 | 4.0688 | 4.1967 | 4.0361 | 4.1242 | 4.0489 | 4.1021 | 4.0453 | 4.0537 | 4.1319 | 3.8670 | 4.1789 | 4.0413 | 3.7940 | +| Vectorization ratio [%] | 64.7217 | 65.8448 | 65.8547 | 65.5270 | 65.3435 | 65.9528 | 65.4245 | 65.6953 | 65.3315 | 65.7638 | 65.6878 | 65.5413 | 65.8699 | 65.5882 | 66.1233 | 66.6024 | ++-------------------------+------------+------------+------------+------------+------------+------------+------------+------------+------------+------------+-------------+-------------+-------------+-------------+-------------+-------------+ + ++------------------------------+------------+-----------+-----------+-----------+ +| Metric | Sum | Min | Max | Avg | ++------------------------------+------------+-----------+-----------+-----------+ +| Runtime (RDTSC) [s] STAT | 17850.4704 | 1115.6544 | 1115.6544 | 1115.6544 | +| Runtime unhalted [s] STAT | 5823.6817 | 355.5116 | 370.6634 | 363.9801 | +| Clock [MHz] STAT | 48007.6661 | 3000.0380 | 3003.9178 | 3000.4791 | +| CPI STAT | 14.4230 | 0.8843 | 0.9059 | 0.9014 | +| DP [MFLOP/s] STAT | 463.9196 | 27.8141 | 30.1708 | 28.9950 | +| AVX DP [MFLOP/s] STAT | 299.6012 | 17.9514 | 19.3436 | 18.7251 | +| Packed [MUOPS/s] STAT | 124.5170 | 7.4632 | 8.0544 | 7.7823 | +| Scalar [MUOPS/s] STAT | 65.0852 | 3.7940 | 4.3903 | 4.0678 | +| Vectorization ratio [%] STAT | 1050.8725 | 64.7217 | 66.6024 | 65.6795 | ++------------------------------+------------+-----------+-----------+-----------+ +``` + +#### Comparison with previous results (`--enable-debug`) and with microbenchmarks + +--- + +**Memory Metrics (`MEM` group, best rank):** + +| Metric | `--enable-debug` | `--enable-ipo` | Change | +| | (4 MPI ranks × 4 OMP threads) | (16 OMP threads) | | +|-------------------------------|-------------------------------|------------------|-----------------------| +| Runtime (RDTSC) [s] | 306.15 | 1025.74 | ~3.35x increase | +| Memory BW [MB/s] | 1515 | 264 | ~5.7x decrease | +| Memory Data Volume [GB] | 464 | 270 | ~42% decrease | +| CPI | 0.84 | 0.90 | Slightly worse | + +- **Observation:** + - With IPO enabled, runtime increases and memory bandwidth drops significantly compared to the debug build. + - CPI is slightly worse, indicating less efficient instruction execution. + - Data volume is lower, possibly due to different run lengths or internal optimizations. + +--- + +**Floating-Point Metrics (`FLOPS_DP` group, best rank):** + +| Metric | SWIFT(`--enable-debug`) | SWIFT(`--enable-ipo`) | Change | +| | (4 MPI ranks × 4 OMP threads) | (16 OMP threads) | | +|------------------|-------------------------------|-----------------------|-----------------------| +| DP MFLOP/s | ~77 | ~29 | ~2.7x decrease | +| Vectorization [%]| ~66 | ~66 | Similar | +| CPI | ~0.84 | ~0.90 | Slightly worse | + +- **Observation:** + - Floating-point throughput drops with IPO enabled. + - Vectorization ratio remains similar. + - CPI is slightly worse. + +--- + +**Comparison with Microbenchmarks:** + +| Metric | SWIFT(`--enable-ipo`) | SWIFT(`--enable-debug`) | likwid-bench triad | likwid-bench peakflops | +| | (16 OMP threads) | (4 MPI ranks × 4 OMP threads) | | | +|----------------|-----------------------|-------------------------------|--------------------|------------------------| +| Memory BW | 264 MB/s | 1515 MB/s | 26614 MB/s | 2468 MB/s | +| DP MFLOP/s | ~29 | ~77 | (not measured) | 4936 | +| Vectorization | ~66% | ~66% | (not measured) | ~100% | +| CPI | 0.90 | 0.84 | 3.13 (triad) | 8.43 (peakflops) | + +- **Observation:** + - Both SWIFT builds achieve only a small fraction of the hardware's peak memory bandwidth and floating-point throughput. + - IPO does not improve performance for this workload; in fact, it reduces both memory and compute throughput. + - Vectorization is much lower than in microbenchmarks. + - CPI is much better than microbenchmarks, but this is typical for memory-bound codes. + +--- + +**Key Takeaways** + +- **IPO did not improve performance** for this configuration; memory and floating-point throughput are lower than with `--enable-debug`. +- **SWIFT remains far from hardware peak** for both memory and compute, with vectorization much lower than microbenchmarks. +- **Optimization potential remains high** for memory access patterns and vectorization in SWIFT. + +--- + +### MAQAO Performance Analysis + +#### Runtime Comparison + +| Tool | Configuration | Runtime [s] | Notes | +|--------------|------------------------------------------------|-------------|-------------------------| +| **MAQAO** | (4 MPI ranks × 4 OMP threads) | 294.31 | 1000 humans, 1000 steps | +| **LIKWID** | `--enable-debug` (4 MPI ranks × 4 OMP threads) | 306.15 | 1000 humans, 100 steps | +| **LIKWID** | `--enable-ipo` (16 OMP threads) | 1025.74 | 1000 humans, 100 steps | + +#### MAQAO Analysis Results + +- **Total Time**: 294.31 seconds +- **Profiled Time**: 239.35 seconds (81.3% coverage) +- **Configuration**: 4 MPI ranks with 4 OpenMP threads each + +#### Key Differences in Methodology + +| Aspect | LIKWID | MAQAO | +|---------------------|----------------------------------|----------------------------------| +| **Scope** | Hardware counter-based profiling | Static + dynamic analysis | +| **Granularity** | Thread/rank level | Loop level | +| **Memory Analysis** | Aggregate bandwidth measurements | Detailed access pattern analysis | +| **Vectorization** | Overall percentage | Per-loop analysis | +| **Output** | Performance counters | Interactive HTML reports | + +#### Performance Insights + +1. **Runtime Efficiency** + - **MAQAO MPI run (294s)** vs **LIKWID debug (306s)**: Similar performance +2. **Analysis Depth** + - **LIKWID** provides quantitative metrics (1515 MB/s memory BW, ~77 MFLOP/s) + - **MAQAO** provides qualitative loop-by-loop optimization guidance +3. **Profiling Coverage** + - **MAQAO** achieved 81.3% profiling coverage + - **LIKWID** measures specific hardware events with 100% coverage +4. **Memory Access Patterns** - MAQAO shows various loops with different memory access patterns: + - Some loops show spans of 652,539-171,205 bytes + - Memory access patterns vary from simple (6 loads) to complex (18 mixed loads/stores) + +#### Complementary Analysis + +The tools complement each other: +- **LIKWID** gives you the **"what"** (quantitative performance) +- **MAQAO** gives you the **"where and why"** (code locations and optimization opportunities) + +--- + +## Intra (shared memory, threads) + +## Inter (MPI) + +### Strong scaling + +### Weak scaling + +### Load balancing diff --git a/examples/HumanMobility/gen.sh b/examples/HumanMobility/gen.sh new file mode 100755 index 000000000..9ba945a8d --- /dev/null +++ b/examples/HumanMobility/gen.sh @@ -0,0 +1,78 @@ +#!/bin/bash + +# Default parameters +BOX_SIZE=100000 +GRID_SIZE=10000 +NUM_HUMANS=1000 + +# Parse command line arguments +while [[ $# -gt 0 ]]; do + case $1 in + -b|--box-size) + BOX_SIZE="$2" + shift 2 + ;; + -g|--grid-size) + GRID_SIZE="$2" + shift 2 + ;; + -n|--num-humans) + NUM_HUMANS="$2" + shift 2 + ;; + -h|--help) + echo "Usage: $0 [OPTIONS]" + echo "Options:" + echo " -b, --box-size BOX_SIZE Box size in meters (default: 100000)" + echo " -g, --grid-size GRID_SIZE Grid size for acceleration field (default: 10000)" + echo " -n, --num-humans NUM_HUMANS Number of humans per dimension (default: 1000)" + echo " -h, --help Show this help message" + exit 0 + ;; + *) + echo "Unknown option: $1" + echo "Use -h or --help for usage information" + exit 1 + ;; + esac +done + +# Determine partition suffix based on SLURM partition +partition_suffix="" +if [[ "${SLURM_JOB_PARTITION}" == "cosma" ]]; then + partition_suffix="-cosma" +fi + +# Basenames for this run (without extensions) +HUMANS=humans-rivers-${NUM_HUMANS}${partition_suffix} +RIVERS=river-rivers-${NUM_HUMANS}${partition_suffix} + +echo "Generation Configuration:" +echo " Box size: ${BOX_SIZE} m" +echo " Grid size: ${GRID_SIZE} cells" +echo " Number of humans: ${NUM_HUMANS}x${NUM_HUMANS}" +echo " Partition: ${SLURM_JOB_PARTITION:-cosma5}" +echo " Output files: ${HUMANS}.hdf5, ${RIVERS}.hdf5" +echo "" + +# Remove previously generated data and images to regenerate the new ones +rm -f ${HUMANS}.hdf5 +rm -f ${RIVERS}.hdf5 + +# Generate acceleration field for river +if [ ! -e ${RIVERS}.hdf5 ] +then + echo "Generating acceleration field for the river..." + python3 makeRivers.py -b ${BOX_SIZE} -g ${GRID_SIZE} -f ${RIVERS}.hdf5 +fi + +# Generate the initial conditions if they are not present. +if [ ! -e ${HUMANS}.hdf5 ] +then + echo "Generating initial conditions for the human mobility box example..." + python3 makeIC.py -t gas -n ${NUM_HUMANS} -b ${BOX_SIZE} -f ${HUMANS}.hdf5 +fi + +echo "Generation complete!" +echo " River file: ${RIVERS}.hdf5" +echo " Humans file: ${HUMANS}.hdf5" \ No newline at end of file diff --git a/examples/HumanMobility/humanMobility.yml b/examples/HumanMobility/humanMobility_template.yml similarity index 50% rename from examples/HumanMobility/humanMobility.yml rename to examples/HumanMobility/humanMobility_template.yml index d97a5ee40..24156c0a5 100644 --- a/examples/HumanMobility/humanMobility.yml +++ b/examples/HumanMobility/humanMobility_template.yml @@ -9,14 +9,14 @@ InternalUnitSystem: # Parameters governing the time integration TimeIntegration: time_begin: 0. # The starting time of the simulation (in internal units). - time_end: 5 #43200 # minutes in a month # The end time of the simulation (in internal units). + time_end: 10 #43200 # minutes in a month # The end time of the simulation (in internal units). dt_min: 1e-7 # minute #1e-5 # The minimal time-step size of the simulation (in internal units). dt_max: 1e-1 #1440 # The maximal time-step size of the simulation (in internal units). # Parameters governing the snapshots Snapshots: - subdir: data - basename: humanMobility # Common part of the name of output files + subdir: . # Current directory (since we cd into ${DATA}) + basename: ${HUMANMOBILITY} # Common part of the name of output files time_first: 0. # Time of the first output (in internal units) delta_time: 1e-2 #1440 # Time difference between consecutive outputs (in internal units) @@ -30,17 +30,36 @@ SPH: CFL_condition: 0.1 # Courant-Friedrich-Levy condition for time integration. Scheduler: - max_top_level_cells: 3 + max_top_level_cells: 16 tasks_per_cell: 64 + # links_per_tasks: 1024 # Parameters related to the initial conditions InitialConditions: - file_name: humans.hdf5 # The file to read + file_name: ../${HUMANS}.hdf5 # The file to read (relative to data directory) periodic: 1 PhysicalConstants: G: 1.0 # Gravitational constant in internal units - + +# Add these gravity parameters +Gravity: + MAC: adaptive # Multipole Acceptance Criterion (adaptive/geometric) + mesh_side_length: 32 # Number of cells in each dimension of the gravity mesh + scale_factor_a: 1.0 # Scale factor for cosmological simulations (1.0 for non-cosmological) + theta: 0.7 # Opening angle for the gravity calculation + theta_cr: 0.5 # Critical opening angle for adaptive MAC + theta_min: 0.0 # Minimum opening angle + max_smoothing_iterations: 3 # Maximum number of iterations for force smoothing + epsilon_fmm: 0.001 # Accuracy parameter for FMM + eta: 0.025 # Constant controlling force accuracy in tree-PM algorithm + comoving_DM_softening: 0.0 # Softening length for gravity in internal units + max_physical_DM_softening: 0.0 # Max physical softening length + comoving_baryon_softening: 1.0 # Softening length for gas particles + max_physical_baryon_softening: 1.0 # Max physical softening for gas + mesh_uses_cell_grad: 1 # Whether to use cell gradients in mesh gravity + periodic: 1 # Use periodic gravity (matches InitialConditions) + # ConstantPotential: # g_cgs: [0., -98, 0.] # Earth acceleration along z-axis (cgs units) @@ -51,5 +70,12 @@ PhysicalConstants: # # timestep_mult: 0.1 # (Optional) The dimensionless constant C in the time-step condition RiverPotential: - position: [5010., 5090.] # y-coordinates of the river (internal units) - mass: 1e6 # "Mass" of the "river" (internal units) + parameter_file: ../${RIVERS}.hdf5 # Path to acceleration field HDF5 file (relative to data directory) + +DomainDecomposition: + trigger: 0.05 # Enable load balancing analysis + initial_type: memory + repartition_type: timecosts + use_fixed_costs: 0 + memory_reports: 1 # Enable memory balance logging + cpu_reports: 1 # Enable CPU balance logging diff --git a/examples/HumanMobility/job.sh b/examples/HumanMobility/job.sh new file mode 100644 index 000000000..be19bb28e --- /dev/null +++ b/examples/HumanMobility/job.sh @@ -0,0 +1,152 @@ +#!/bin/bash +#SBATCH -A durham # Account +#SBATCH -t 12:00:00 # Time limit hrs:min:sec +#SBATCH --mail-type=END +##SBATCH --mail-user= +# Note: SLURM partition and other parameters are passed from submit.sh + +# Check if an argument was provided +if [ $# -lt 1 ]; then + echo "Usage: $0 [--gen|--vis|--map|--run|--likwid|--maqao|--aps|--vtune|--advisor|--inspector|--scorep] [mode_specific_params]" + exit 1 +fi + +mode=$1 +shift # Remove mode from arguments + +case $mode in + --gen|--vis|--map|--run|--likwid|--maqao|--aps|--vtune|--advisor|--inspector|--scorep) + ;; + *) + echo "Invalid argument. Use --gen, --vis, --map, --run, --likwid, --maqao, --aps, --vtune, --advisor, --inspector, or --scorep" + exit 1 + ;; +esac + +# Store remaining arguments as mode-specific parameters +mode_params="$*" + +# Detect partition and set hardware description +if [[ "${SLURM_JOB_PARTITION}" == "cosma" ]]; then + hardware_desc="COSMA (legacy hardware)" + intel_comp_version="intel_comp/2024.2.0" # Use older Intel compiler for compatibility +else + hardware_desc="COSMA5 (modern hardware)" + intel_comp_version="intel_comp/2025.0.1" # Use latest Intel compiler +fi + +# Display current job configuration +echo "Job Configuration:" +echo " Mode: $mode" +echo " Partition: $hardware_desc" +echo " Nodes: ${SLURM_JOB_NUM_NODES:-1}" +echo " MPI Tasks: ${SLURM_NTASKS:-1}" +echo " CPUs per Task: ${SLURM_CPUS_PER_TASK:-16}" +echo " Memory per Node: ${SLURM_MEM_PER_NODE:-64G}" + +# Show mode-specific parameters if provided +if [ -n "$mode_params" ]; then + case $mode in + --gen) + echo " Generation Parameters: $mode_params" + ;; + --vis) + echo " Visualization Parameters: $mode_params" + ;; + --map) + echo " Map Parameters: $mode_params" + ;; + --run) + echo " Run Parameters: $mode_params" + ;; + --likwid|--maqao|--aps|--vtune|--advisor|--inspector|--scorep) + echo " Performance Analysis Parameters: $mode_params" + ;; + esac +fi + +# Common module loading +module purge +module load cosma +module load $intel_comp_version +if [[ "$intel_comp_version" == "intel_comp/2025.0.1" ]]; then + module load umf +fi +module load compiler-rt tbb compiler mpi +# module load openmpi/5.0.3/ + +# Set MPI environment variables +export I_MPI_PMI_LIBRARY=/usr/lib64/libpmi.so +export I_MPI_FABRICS=shm:ofi + +module load python +module load fftw/3.3.10 +module load gsl +module load parmetis/4.0.3-64bit +module load parallel_hdf5/1.14.4 +module load sundials/5.8.0_c8_single + +# Load analysis-specific modules and execute the appropriate script +case $mode in + --gen) + echo "Running generation with parameters: $mode_params" + ./gen.sh $mode_params + ;; + --vis) + module load ffmpeg + echo "Running visualization with parameters: $mode_params" + ./vis.sh $mode_params + ;; + --map) + echo "Running map generation with parameters: $mode_params" + ./map.sh $mode_params + ;; + --run) + echo "Running simulation with parameters: $mode_params" + ./run.sh $mode_params + ;; + --likwid) + # Load likwid profiler + module load likwid/5.4.1 + ./run_perf.sh --likwid + ;; + --maqao) + # Load Maqao profiler + module load maqao + ./run_perf.sh --maqao + ;; + --aps) + if [[ "${SLURM_JOB_PARTITION}" == "cosma" ]]; then + echo "Warning: Intel APS 2025 may not be compatible with legacy COSMA hardware" + fi + # Load APS profiler + source /cosma/local/intel/oneAPI_2025.0.1/vtune/2025.0/vtune-vars.sh + ./run_perf.sh --aps + ;; + --vtune) + if [[ "${SLURM_JOB_PARTITION}" == "cosma" ]]; then + echo "Warning: Intel VTune 2025 may not be compatible with legacy COSMA hardware" + fi + # Load VTune profiler + source /cosma/local/intel/oneAPI_2025.0.1/vtune/2025.0/vtune-vars.sh + ./run_perf.sh --vtune + ;; + --advisor) + if [[ "${SLURM_JOB_PARTITION}" == "cosma" ]]; then + echo "Warning: Intel Advisor 2025 may not be compatible with legacy COSMA hardware" + fi + # Load Intel Advisor 2025.0 + source /cosma/local/intel/oneAPI_2025.0.1/advisor/2025.0/advisor-vars.sh + ./run_perf.sh --advisor + ;; + --inspector) + # Load Intel Inspector + source /cosma/local/intel/oneAPI_2023.2.0/inspector/2023.2.0/inspxe-vars.sh + ./run_perf.sh --inspector + ;; + --scorep) + # Load Score-P for detailed instrumentation + module load scorep/8.4 + ./run_perf.sh --scorep + ;; +esac diff --git a/examples/HumanMobility/makeIC.py b/examples/HumanMobility/makeIC.py index bbe9e1eae..2e0a7e7ea 100644 --- a/examples/HumanMobility/makeIC.py +++ b/examples/HumanMobility/makeIC.py @@ -22,6 +22,7 @@ import numpy as np import h5py as h5 import write_gadget as wg +import time class Humans(object): @@ -58,6 +59,7 @@ def calculate_velocities(self, angle=0): v_x = (np.random.rand(self.nhumans)-0.5)*1000 #np.zeros(self.nhumans) # v_y = (np.random.rand(self.nhumans)-0.5)*1000 #np.zeros(self.nhumans) # v_z = np.zeros(self.nhumans) + # print("Velocity means:", v_x.mean(), v_y.mean(), v_z.mean()) self.velocities = np.array([v_x, v_y, v_z]).T @@ -82,21 +84,43 @@ def generate_ids(self): return self.ids - def save_to_gadget(self, filename, boxsize=10000): + def save_to_gadget(self, filename, boxsize=10000, type="particles"): """ Save the human data to a GADGET .hdf5 file. Uses the internal options, but you must specify a filename. + + Parameters: + filename (str): The output HDF5 filename + boxsize (float): Size of the simulation box + type (str): Type of particles to write - either "gas" or "particles" """ + # Validate particle type + if type not in ["gas", "particles"]: + raise ValueError('type must be either "gas" or "particles"') + + # Map particle type to index + type_index = 0 if type == "gas" else 1 + + # Create arrays with the correct particle type index + np_total = np.zeros(6, dtype=int) + np_total[type_index] = self.nhumans + + mass_table = np.zeros(6) + mass_table[type_index] = self.humanmass + + # type(boxsize) + print("boxsize:", float(boxsize)) + with h5.File(filename, "w") as handle: wg.write_header( handle, - boxsize=[boxsize, boxsize], + boxsize=[float(boxsize), float(boxsize)], flag_entropy=0, - np_total=np.array([0, self.nhumans, 0, 0, 0, 0]), + np_total=np_total, np_total_hw=np.array([0, 0, 0, 0, 0, 0]), other={ - "MassTable": np.array([0, self.humanmass, 0, 0, 0, 0]), + "MassTable": mass_table, "Time": 0, "Dimension": 2, "Flag_Entropy_ICs": 0, @@ -111,7 +135,7 @@ def save_to_gadget(self, filename, boxsize=10000): wg.write_block( handle, - 1, # gas, # particles, dark matter + type_index, # gas, dark matter particles self.positions, self.velocities, self.ids, @@ -129,19 +153,20 @@ def gen_humans_grid(meta): Generates humans on a grid and returns a filled Humans object. """ humans = Humans(meta) - range = (0, meta["boxsize"]) - centre_of_ring = [meta["boxsize"] / 2.0] * 3 + positions = (0, meta["boxsize"]) # -0.5*meta["boxsize"] + # centre_of_ring = [meta["boxsize"] * 0.0] * 3 # Because we are using a uniform grid we actually use the same x and y - # range for the initial human setup. - step = (range[1] - range[0]) / meta["nhumans"] + # positions for the initial human setup. + width = positions[1] - positions[0] + step = width / meta["nhumans"] - x_values = np.arange(0, range[1] - range[0], step) + x_values = np.arange(0, width, step, dtype=float) # -0.5*width # These are 2d arrays which isn't actually that helpful. x, y = np.meshgrid(x_values, x_values) - x = x.flatten() + centre_of_ring[0] - (range[1] - range[0]) / 2 + np.random.rand(humans.nhumans) * 0.001 - y = y.flatten() + centre_of_ring[1] - (range[1] - range[0]) / 2 + np.random.rand(humans.nhumans) * 0.001 + x = x.flatten() + np.random.rand(humans.nhumans) * 5 + y = y.flatten() + np.random.rand(humans.nhumans) * 5 z = np.zeros(humans.nhumans) # z = np.zeros_like(x) + meta["boxsize"] / 2 @@ -257,6 +282,20 @@ def gen_humans_grid(meta): default=10000, ) + PARSER.add_argument( + "-t", + "--type", + help=""" + Type of particles to use - either 'gas' or 'particles'. + 'gas' will save humans as SPH particles (PartType0), + 'particles' will save them as dark matter particles (PartType1). + Default: gas + """, + required=False, + choices=['gas', 'particles'], + default='gas', + ) + ### --- ### --- Argument Parsing --- ### --- ### ARGS = vars(PARSER.parse_args()) @@ -285,104 +324,19 @@ def gen_humans_grid(meta): "boxsize": float(ARGS["boxsize"]), } + start_time = time.time() HUMANS = gen_humans(META) - - HUMANS.save_to_gadget(filename=ARGS["filename"], boxsize=ARGS["boxsize"]) - + gen_time = time.time() + print(f"Human data generation took {gen_time - start_time:.2f} seconds.") + + # For SPH gas particles (PartType0) or dark matter particles (PartType1) + HUMANS.save_to_gadget( + filename=ARGS["filename"], + boxsize=ARGS["boxsize"], + type=ARGS["type"] + ) + end_time = time.time() + print(f"HDF5 writing took {end_time - gen_time:.2f} seconds.") + print(f"Total time: {end_time - start_time:.2f} seconds.") print("Initial condition generated") - -################################################# - -# # Generates a SWIFT IC file with ... - -# # Parameters -# periodic = 0 # 1 For periodic box -# boxSize = 10 # 1 km -# rho = 200 # Population density in code units ? -# T = 1 # Initial intensity of human motion (how many people are moving in a box or percentage?) -# gamma = 5.0 / 3.0 # Gas adiabatic index -# fileName = "mobilityBox.hdf5" -# # --------------------------------------------------- - -# # defines some constants -# # need to be changed in plotTemperature.py too -# h_frac = 0.76 -# mu = 4.0 / (1.0 + 3.0 * h_frac) - -# m_h_cgs = 1.67e-24 -# k_b_cgs = 1.38e-16 - -# # defines units -# unit_length = 1 # 1m -# unit_mass = 1 # a unit mass of 1 particle-human -# unit_time = 1 # 1 s ? - -# # Read id, position and h from glass -# glass = h5.File("humans.hdf5", "r") -# ids = glass["/PartType0/ParticleIDs"][:] -# pos = glass["/PartType0/Coordinates"][:, :] * boxSize -# h = glass["/PartType0/SmoothingLength"][:] * boxSize - -# # Compute basic properties - -# # need to define `pos` - -# numHum = np.size(pos) // 2 -# mass = boxSize ** 2 * rho # number of humans in a unit of territory -# internalEnergy = k_b_cgs * T * mu / ((gamma - 1.0) * m_h_cgs) -# internalEnergy *= (unit_time / unit_length) ** 2 - -# # File -# f = h5.File(fileName, "w") - -# # Header -# grp = f.create_group("/Header") -# grp.attrs["BoxSize"] = boxSize -# grp.attrs["NumHum_Total"] = [numHum, 0, 0, 0, 0, 0] -# grp.attrs["NumHum_Total_HighWord"] = [0, 0, 0, 0, 0, 0] -# grp.attrs["NumHum_ThisFile"] = [numHum, 0, 0, 0, 0, 0] -# grp.attrs["Time"] = 0.0 -# grp.attrs["NumFilesPerSnapshot"] = 1 -# grp.attrs["MassTable"] = [0.0, 0.0, 0.0, 0.0, 0.0, 0.0] -# # grp.attrs["Flag_Entropy_ICs"] = 0 - -# # Runtime parameters -# grp = f.create_group("/RuntimePars") -# grp.attrs["PeriodicBoundariesOn"] = periodic - -# # Units -# grp = f.create_group("/Units") -# grp.attrs["Unit length in cgs (U_L)"] = unit_length -# grp.attrs["Unit mass in cgs (U_M)"] = unit_mass -# grp.attrs["Unit time in cgs (U_t)"] = unit_time -# grp.attrs["Unit current in cgs (U_I)"] = 1.0 -# grp.attrs["Unit temperature in cgs (U_T)"] = 1.0 - -# # Particle group -# grp = f.create_group("/PartType0") # humans - -# v = np.zeros((numHum, 2)) -# ds = grp.create_dataset("Velocities", (numHum, 2), "f") -# ds[()] = v - -# m = np.full((numHum, 1), mass) -# ds = grp.create_dataset("Masses", (numHum, 1), "f") -# ds[()] = m - -# h = np.reshape(h, (numHum, 1)) -# ds = grp.create_dataset("SmoothingLength", (numHum, 1), "f") -# ds[()] = h - -# u = np.full((numHum, 1), internalEnergy) -# ds = grp.create_dataset("InternalEnergy", (numHum, 1), "f") -# ds[()] = u - -# ids = np.reshape(ids, (numHum, 1)) -# ds = grp.create_dataset("ParticleIDs", (numHum, 1), "L") -# ds[()] = ids - -# ds = grp.create_dataset("Coordinates", (numHum, 2), "d") -# ds[()] = pos - -# f.close() diff --git a/examples/HumanMobility/makeRiver.py b/examples/HumanMobility/makeRiver.py new file mode 100644 index 000000000..0c3bf1d50 --- /dev/null +++ b/examples/HumanMobility/makeRiver.py @@ -0,0 +1,151 @@ +import numpy as np +import h5py as h5 +import argparse as ap +from scipy.ndimage import gaussian_filter + +def generate_river(box_size, river_width=60.0): + """Generate a horizontal river with constant width""" + + # Generate river centerline (straight line from west to east) + x_center = np.linspace(0, box_size[0], 1000) + y_center = np.ones_like(x_center) * 5050.0 # Changed from box_size[1]/2 to 5050 + + # Generate river banks by offsetting from centerline + half_width = river_width / 2.0 + left_bank_x = x_center + left_bank_y = y_center - half_width + right_bank_x = x_center + right_bank_y = y_center + half_width + + return x_center, y_center, left_bank_x, left_bank_y, right_bank_x, right_bank_y + +def calculate_river_acceleration(x, y, left_bank_x, left_bank_y, right_bank_x, right_bank_y, mass, distance): + """Calculate acceleration at point (x,y) due to river banks + + For horizontal river: + - Inside river: acceleration proportional to distance from centerline + - Within 'distance' from banks: acceleration proportional to distance constant + - Outside: acceleration proportional to actual distance from closest bank + """ + + # For horizontal river, we only need y-distances + dy_left = y - left_bank_y[0] # y-coordinate is constant for each bank + dy_right = y - right_bank_y[0] + river_center = (left_bank_y[0] + right_bank_y[0]) / 2.0 + + # Calculate acceleration + ax = 0.0 + ay = 0.0 + + # Inside river check + if dy_left >= 0 and dy_right <= 0: + # Inside river - acceleration proportional to distance from centerline + dy_center = y - river_center + r = distance # Use constant distance for magnitude + rinv3 = 1.0 / (r * r * r) + # Direction away from centerline + ay = mass * np.sign(dy_center) * distance * rinv3 + else: + # Outside river or near banks + # Find distance to closest bank + if abs(dy_left) < abs(dy_right): + # Closer to left bank + if abs(dy_left) <= distance: + # Within distance constant from bank - use constant force + r = distance + rinv3 = 1.0 / (r * r * r) + ay = mass * np.sign(dy_left) * distance * rinv3 + else: + # Beyond distance constant - use actual distance + r = abs(dy_left) + rinv3 = 1.0 / (r * r * r) + ay = mass * dy_left * rinv3 + else: + # Closer to right bank + if abs(dy_right) <= distance: + # Within distance constant from bank - use constant force + r = distance + rinv3 = 1.0 / (r * r * r) + ay = mass * np.sign(dy_right) * distance * rinv3 + else: + # Beyond distance constant - use actual distance + r = abs(dy_right) + rinv3 = 1.0 / (r * r * r) + ay = mass * dy_right * rinv3 + + return ax, ay + +def main(): + # Parse arguments + parser = ap.ArgumentParser() + parser.add_argument("-f", "--file", type=str, default="river.hdf5") + parser.add_argument("-s", "--seed", type=int, default=42, + help="Random seed for river generation") + args = parser.parse_args() + + # Set random seed for reproducibility + np.random.seed(args.seed) + + # Parameters + box_size = [10000., 10000.] # meters + mass = 1e5 # "mass" of the river + distance = 1.0 # minimal distance from river + river_width = 60.0 # width of the river + + # Grid parameters + grid_size = [1001, 1001] # number of cells + 1 + + # Generate straight river + x_center, y_center, left_bank_x, left_bank_y, right_bank_x, right_bank_y = \ + generate_river(box_size, river_width=river_width) + + # Create acceleration field arrays + ax = np.zeros(grid_size, dtype=np.float32) + ay = np.zeros(grid_size, dtype=np.float32) + + # Generate grid coordinates + x_coords = np.linspace(0, box_size[0], grid_size[0]) + y_coords = np.linspace(0, box_size[1], grid_size[1]) + + # Calculate acceleration field at each grid point + print("Calculating acceleration field...") + for i, x in enumerate(x_coords): + if i % 100 == 0: # Progress indicator + print(f"Processing row {i}/{grid_size[0]}") + for j, y in enumerate(y_coords): + ax[i,j], ay[i,j] = calculate_river_acceleration( + x, y, left_bank_x, left_bank_y, right_bank_x, right_bank_y, + mass, distance + ) + + # Optional: Smooth the acceleration field + # print("Smoothing acceleration field...") + # ax = gaussian_filter(ax, sigma=1.0) + # ay = gaussian_filter(ay, sigma=1.0) + + # Save to HDF5 file + print("Saving to HDF5 file...") + with h5.File(args.file, 'w') as f: + # Save acceleration field + f.create_dataset("AccelerationField/ax", data=ax) + f.create_dataset("AccelerationField/ay", data=ay) + + # Save river geometry for visualization + f.create_dataset("RiverGeometry/centerline_x", data=x_center) + f.create_dataset("RiverGeometry/centerline_y", data=y_center) + f.create_dataset("RiverGeometry/left_bank_x", data=left_bank_x) + f.create_dataset("RiverGeometry/left_bank_y", data=left_bank_y) + f.create_dataset("RiverGeometry/right_bank_x", data=right_bank_x) + f.create_dataset("RiverGeometry/right_bank_y", data=right_bank_y) + + # Save metadata + f.create_group("Header") + f["Header"].attrs["BoxSize"] = np.array(box_size, dtype=np.float64) + f["Header"].attrs["GridSize"] = np.array(grid_size, dtype=np.int32) + f["Header"].attrs["Mass"] = np.array(mass, dtype=np.float64) + f["Header"].attrs["Distance"] = np.array(distance, dtype=np.float64) + f["Header"].attrs["RiverWidth"] = np.array(river_width, dtype=np.float64) + f["Header"].attrs["RandomSeed"] = np.array(args.seed, dtype=np.int32) # Add this line + +if __name__ == "__main__": + main() diff --git a/examples/HumanMobility/makeRivers.py b/examples/HumanMobility/makeRivers.py new file mode 100644 index 000000000..ae86ee9e9 --- /dev/null +++ b/examples/HumanMobility/makeRivers.py @@ -0,0 +1,339 @@ +import numpy as np +import h5py as h5 +import argparse as ap +from scipy.ndimage import gaussian_filter +from scipy.interpolate import splprep, splev +import concurrent.futures +import time +import os + +def generate_meandering_river(box_size, num_control_points=8, river_width=60.0, randomness=0.15): + """Generate a meandering river using control points and spline interpolation""" + + print("Generating meandering river...") + + # Generate random control points for river centerline + # Start from left side (west) + x_controls = np.linspace(0, box_size[0], num_control_points) + + # Add random vertical displacement to control points (keeping ends fixed) + y_controls = np.zeros(num_control_points) + y_controls[1:-1] = box_size[1]/2 + randomness * box_size[1] * np.random.randn(num_control_points-2) + y_controls[0] = box_size[1]/2 # Fix start point + y_controls[-1] = box_size[1]/2 # Fix end point + + # Fit a spline through control points + tck, u = splprep([x_controls, y_controls], s=0, k=3) + + # Generate points along the spline for smooth river + t_fine = np.linspace(0, 1, 1000) + x_center, y_center = splev(t_fine, tck) + + # Calculate tangent vectors along the river + dx = np.gradient(x_center) + dy = np.gradient(y_center) + + # Normalize tangent vectors + norm = np.sqrt(dx*dx + dy*dy) + dx /= norm + dy /= norm + + # Calculate normal vectors (perpendicular to tangent) + normal_x = -dy + normal_y = dx + + # Generate river banks by offsetting perpendicular to centerline + half_width = river_width / 2.0 + left_bank_x = x_center - normal_x * half_width + left_bank_y = y_center - normal_y * half_width + right_bank_x = x_center + normal_x * half_width + right_bank_y = y_center + normal_y * half_width + + return x_center, y_center, left_bank_x, left_bank_y, right_bank_x, right_bank_y + +def calculate_river_acceleration(x, y, left_bank_x, left_bank_y, right_bank_x, right_bank_y, mass, distance): + """Calculate acceleration at point (x,y) due to river banks + + For meandering river: + - Inside river: acceleration proportional to distance from centerline + - Within 'distance' from banks: acceleration proportional to distance constant + - Outside: acceleration proportional to actual distance from closest bank + """ + + # Find distances and closest points on both banks + left_dists = np.sqrt((x - left_bank_x)**2 + (y - left_bank_y)**2) + right_dists = np.sqrt((x - right_bank_x)**2 + (y - right_bank_y)**2) + + left_idx = np.argmin(left_dists) + right_idx = np.argmin(right_dists) + min_left_dist = left_dists[left_idx] + min_right_dist = right_dists[right_idx] + + # Calculate vectors from banks to point + dx_left = x - left_bank_x[left_idx] + dy_left = y - left_bank_y[left_idx] + dx_right = x - right_bank_x[right_idx] + dy_right = y - right_bank_y[right_idx] + + # Calculate normal vectors to check if point is between banks + normal_x = right_bank_x[right_idx] - left_bank_x[left_idx] + normal_y = right_bank_y[right_idx] - left_bank_y[left_idx] + norm = np.sqrt(normal_x*normal_x + normal_y*normal_y) + normal_x /= norm + normal_y /= norm + + ax = 0.0 + ay = 0.0 + + # Inside river check (if between banks) + dot_left = dx_left*normal_x + dy_left*normal_y + dot_right = dx_right*normal_x + dy_right*normal_y + if dot_left >= 0 and dot_right <= 0: # This checks if point is between banks + # Inside river - acceleration proportional to distance from centerline + center_x = (left_bank_x[left_idx] + right_bank_x[right_idx]) / 2 + center_y = (left_bank_y[left_idx] + right_bank_y[right_idx]) / 2 + dx = x - center_x + dy = y - center_y + r = distance # Use constant distance for magnitude + rinv3 = 1.0 / (r * r * r) + # Direction away from centerline + norm = np.sqrt(dx*dx + dy*dy) + if norm > 0: + ax = mass * (dx/norm) * distance * rinv3 + ay = mass * (dy/norm) * distance * rinv3 + else: + # Outside river or near banks + # Find closest bank + if min_left_dist < min_right_dist: + # Closer to left bank + dx = x - left_bank_x[left_idx] + dy = y - left_bank_y[left_idx] + if min_left_dist <= distance: + # Within distance constant from bank - use constant force + r = distance + rinv3 = 1.0 / (r * r * r) + norm = np.sqrt(dx*dx + dy*dy) + ax = mass * (dx/norm) * distance * rinv3 + ay = mass * (dy/norm) * distance * rinv3 + else: + # Beyond distance constant - use actual distance + r = min_left_dist + rinv3 = 1.0 / (r * r * r) + norm = np.sqrt(dx*dx + dy*dy) + ax = mass * (dx/norm) * r * rinv3 + ay = mass * (dy/norm) * r * rinv3 + else: + # Closer to right bank + dx = x - right_bank_x[right_idx] + dy = y - right_bank_y[right_idx] + if min_right_dist <= distance: + # Within distance constant from bank - use constant force + r = distance + rinv3 = 1.0 / (r * r * r) + norm = np.sqrt(dx*dx + dy*dy) + ax = mass * (dx/norm) * distance * rinv3 + ay = mass * (dy/norm) * distance * rinv3 + else: + # Beyond distance constant - use actual distance + r = min_right_dist + rinv3 = 1.0 / (r * r * r) + norm = np.sqrt(dx*dx + dy*dy) + ax = mass * (dx/norm) * r * rinv3 + ay = mass * (dy/norm) * r * rinv3 + + return ax, ay + +def compute_row_block(i, x, y_coords, river_segments, mass, distance): + ax_row = np.zeros(len(y_coords), dtype=np.float32) + ay_row = np.zeros(len(y_coords), dtype=np.float32) + for j, y in enumerate(y_coords): + min_dist = float('inf') + best_seg = None + for seg in river_segments: + seg_y = np.mean(seg["centerline_y"]) + seg_x = np.mean(seg["centerline_x"]) + dist = np.sqrt((x - seg_x)**2 + (y - seg_y)**2) + if dist < min_dist: + min_dist = dist + best_seg = seg + ax_row[j], ay_row[j] = calculate_river_acceleration( + x, y, + best_seg["left_bank_x"], best_seg["left_bank_y"], + best_seg["right_bank_x"], best_seg["right_bank_y"], + mass, distance + ) + return i, ax_row, ay_row + +def main(): + # Parse arguments + parser = ap.ArgumentParser() + parser.add_argument("-f", "--file", type=str, default="river.hdf5", + help="Output HDF5 file name") + parser.add_argument("-s", "--seed", type=int, default=42, + help="Random seed for river generation") + parser.add_argument("-b", "--box-size", type=float, default=10000., + help="Box size in meters (square domain)") + parser.add_argument("-g", "--grid-size", type=int, default=1000, + help="Number of grid cells in each dimension + 1") + args = parser.parse_args() + + # Set random seed for reproducibility + np.random.seed(args.seed) + + # Parameters + max_box = min(args.box_size, 10000.0) + box_size = [max_box, max_box] # restrict to [0, min(args.box_size, 10000)] + mass = 1e4 # "mass" of the river + distance = 1.0 # minimal distance from river + river_width = 200.0 # width of the river + + # Grid parameters + grid_size = [args.grid_size+1, args.grid_size+1] # square grid + + river_segments = [] + block_size = 10000.0 + start_time = time.time() + if args.box_size > block_size: + n_repeat_x = int(np.ceil(args.box_size / block_size)) + n_repeat_y = int(np.ceil(args.box_size / block_size)) + + # Generate the base river only once in the bottom left corner + base_box_size = [block_size, block_size] + x_c, y_c, l_x, l_y, r_x, r_y = generate_meandering_river(base_box_size, river_width=river_width) + + # Replicate the river along X and Y axes + for j in range(n_repeat_y): + for i in range(n_repeat_x): + offset_x = i * block_size + offset_y = j * block_size + # Only keep points within the block (should always be true for the base river) + mask = (x_c >= 0) & (x_c <= block_size) + river_segments.append({ + "centerline_x": x_c[mask] + offset_x, + "centerline_y": y_c[mask] + offset_y, + "left_bank_x": l_x[mask] + offset_x, + "left_bank_y": l_y[mask] + offset_y, + "right_bank_x": r_x[mask] + offset_x, + "right_bank_y": r_y[mask] + offset_y, + }) + box_size = [args.box_size, args.box_size] + else: + x_center, y_center, left_bank_x, left_bank_y, right_bank_x, right_bank_y = \ + generate_meandering_river(box_size, river_width=river_width) + river_segments.append({ + "centerline_x": x_center, + "centerline_y": y_center, + "left_bank_x": left_bank_x, + "left_bank_y": left_bank_y, + "right_bank_x": right_bank_x, + "right_bank_y": right_bank_y, + }) + end_time = time.time() + print(f"River generation took {end_time - start_time:.2f} seconds.") + + # Extended information output + print(f"Block size: {block_size} x {block_size}") + print(f"Grid size in blocks: {n_repeat_x} x {n_repeat_y}" if args.box_size > block_size else "Grid size in blocks: 1 x 1") + print(f"Full domain size: {box_size[0]} x {box_size[1]} (meters)") + print(f"Full grid size: {grid_size[0]} x {grid_size[1]}") + + # Calculate number of blocks + n_repeat_x = int(np.ceil(args.box_size / block_size)) + n_repeat_y = int(np.ceil(args.box_size / block_size)) + + # Grid parameters + grid_size = [args.grid_size+1, args.grid_size+1] # e.g. 10001 x 10001 + + # Calculate block grid size (number of points per block, including overlap) + block_grid_size_x = (grid_size[0] - 1) // n_repeat_x + 1 + block_grid_size_y = (grid_size[1] - 1) // n_repeat_y + 1 + block_grid_size = [block_grid_size_x, block_grid_size_y] + + # Generate grid coordinates for the block + x_block = np.linspace(0, block_size, block_grid_size[0]) + y_block = np.linspace(0, block_size, block_grid_size[1]) + + # Create acceleration field arrays for a single block + ax_block = np.zeros(block_grid_size, dtype=np.float32) + ay_block = np.zeros(block_grid_size, dtype=np.float32) + + # Calculate acceleration field for the bottom-left block + print("Calculating acceleration field for bottom-left block...") + accel_start_time = time.time() + + # Limit the number of worker processes to avoid "too many open files" error + # Use at most 16 processes or the number of CPU cores, whichever is smaller + max_workers = min(16, os.cpu_count() or 1) + + with concurrent.futures.ProcessPoolExecutor(max_workers=max_workers) as executor: + futures = [] + for i, x in enumerate(x_block): + if(i % 100 == 0): + print(f"Processing row {i+1}/{block_grid_size[0]}") + # Submit tasks to the executor + futures.append(executor.submit(compute_row_block, i, x, y_block, river_segments, mass, distance)) + + for future in concurrent.futures.as_completed(futures): + i, ax_row, ay_row = future.result() + ax_block[i, :] = ax_row + ay_block[i, :] = ay_row + accel_end_time = time.time() + print(f"Acceleration field calculation for block took {accel_end_time - accel_start_time:.2f} seconds.") + + # Now assemble the full grid, avoiding duplicated edges + ax = np.zeros(grid_size, dtype=np.float32) + ay = np.zeros(grid_size, dtype=np.float32) + + for i in range(n_repeat_x): + for j in range(n_repeat_y): + # Compute start and end indices for this block in the full grid + x_start = i * (block_grid_size_x - 1) + x_end = x_start + block_grid_size_x + y_start = j * (block_grid_size_y - 1) + y_end = y_start + block_grid_size_y + + # For the last block, ensure we don't go out of bounds + if x_end > grid_size[0]: + x_end = grid_size[0] + if y_end > grid_size[1]: + y_end = grid_size[1] + + # Compute the corresponding slice in the block grid + bx_start = 0 + bx_end = x_end - x_start + by_start = 0 + by_end = y_end - y_start + + ax[x_start:x_end, y_start:y_end] = ax_block[bx_start:bx_end, by_start:by_end] + ay[x_start:x_end, y_start:y_end] = ay_block[bx_start:bx_end, by_start:by_end] + + # Save to HDF5 file + print("Saving to HDF5 file...") + with h5.File(args.file, 'w') as f: + # Save acceleration field + f.create_dataset("AccelerationField/ax", data=ax) + f.create_dataset("AccelerationField/ay", data=ay) + + # Save river geometry for visualization as a collection + river_group = f.create_group("RiverGeometry") + for idx, seg in enumerate(river_segments): + grp = river_group.create_group(f"river_{idx}") + grp.create_dataset("centerline_x", data=seg["centerline_x"]) + grp.create_dataset("centerline_y", data=seg["centerline_y"]) + grp.create_dataset("left_bank_x", data=seg["left_bank_x"]) + grp.create_dataset("left_bank_y", data=seg["left_bank_y"]) + grp.create_dataset("right_bank_x", data=seg["right_bank_x"]) + grp.create_dataset("right_bank_y", data=seg["right_bank_y"]) + river_group.attrs["num_rivers"] = len(river_segments) + + # Save metadata + f.create_group("Header") + f["Header"].attrs["BoxSize"] = np.array(box_size, dtype=np.float64) + f["Header"].attrs["GridSize"] = np.array(grid_size, dtype=np.int32) + f["Header"].attrs["Mass"] = np.array(mass, dtype=np.float64) + f["Header"].attrs["Distance"] = np.array(distance, dtype=np.float64) + f["Header"].attrs["RiverWidth"] = np.array(river_width, dtype=np.float64) + f["Header"].attrs["RandomSeed"] = np.array(args.seed, dtype=np.int32) + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/examples/HumanMobility/map.sh b/examples/HumanMobility/map.sh new file mode 100755 index 000000000..9956e59c9 --- /dev/null +++ b/examples/HumanMobility/map.sh @@ -0,0 +1,102 @@ +#!/bin/bash + +# Default parameters +NUM_HUMANS=1000 +min_x=0 +max_x=100000 +min_y=0 +max_y=100000 + +# Parse command line arguments +dir_suffix="" +while [[ $# -gt 0 ]]; do + case $1 in + --num-humans) + NUM_HUMANS="$2" + shift 2 + ;; + -d|--dir-suffix) + dir_suffix="$2" + shift 2 + ;; + --min-x) + min_x="$2" + shift 2 + ;; + --max-x) + max_x="$2" + shift 2 + ;; + --min-y) + min_y="$2" + shift 2 + ;; + --max-y) + max_y="$2" + shift 2 + ;; + -h|--help) + echo "Usage: $0 [OPTIONS]" + echo "Options:" + echo " --num-humans NUM_HUMANS Number of humans per dimension (default: 1000)" + echo " -d, --dir-suffix Suffix part of directory path to read data from" + echo " --min-x Minimum x coordinate for map bounds (default: 0)" + echo " --max-x Maximum x coordinate for map bounds (default: 100000)" + echo " --min-y Minimum y coordinate for map bounds (default: 0)" + echo " --max-y Maximum y coordinate for map bounds (default: 100000)" + echo " -h, --help Show this help message" + exit 0 + ;; + *) + echo "Unknown option: $1" + echo "Use -h or --help for usage information" + exit 1 + ;; + esac +done + +# If no dir_suffix provided, auto-generate from SLURM parameters +if [ -z "$dir_suffix" ]; then + nodes=${SLURM_JOB_NUM_NODES:-1} + ntasks=${SLURM_NTASKS:-1} + cpus_per_task=${SLURM_CPUS_PER_TASK:-16} + + # Determine partition suffix + partition_suffix="" + if [[ "${SLURM_JOB_PARTITION}" == "cosma" ]]; then + partition_suffix="-cosma" + fi + + dir_suffix="${nodes}-${ntasks}-${cpus_per_task}${partition_suffix}" +else + # Need to determine partition suffix for filename generation + partition_suffix="" + if [[ "${SLURM_JOB_PARTITION}" == "cosma" ]] || [[ "$dir_suffix" == *"-cosma" ]]; then + partition_suffix="-cosma" + fi +fi + +# Basenames for this run (without extensions) +RIVERS=river-rivers-${NUM_HUMANS}${partition_suffix} +IMAGES=images-rivers-${dir_suffix} + +echo "Map Generation Configuration:" +echo " Directory suffix: ${dir_suffix}" +echo " Images directory: ${IMAGES}" +echo " Bounds: x=[${min_x}, ${max_x}], y=[${min_y}, ${max_y}]" + +# Create images directory if it doesn't exist +mkdir -p ${IMAGES} + +# Remove previously generated river images +rm -f ${IMAGES}/rivers.png 2>/dev/null + +echo "Generating river map..." +echo " River file: ${RIVERS}.hdf5" +echo " Output image: ${IMAGES}/rivers.png" +echo " Bounds: x[${min_x}, ${max_x}], y[${min_y}, ${max_y}]" + +# Plot only the rivers +python3 plot_rivers.py ${RIVERS}.hdf5 ${IMAGES}/rivers.png ${min_x} ${max_x} ${min_y} ${max_y} + +echo "Map generation complete! Image saved to: ${IMAGES}/rivers.png" \ No newline at end of file diff --git a/examples/HumanMobility/pa_vis.py b/examples/HumanMobility/pa_vis.py new file mode 100644 index 000000000..0b2442ccc --- /dev/null +++ b/examples/HumanMobility/pa_vis.py @@ -0,0 +1,811 @@ +#!/usr/bin/env python3 +""" +Performance Analysis Visualization (pa_vis.py) + +Creates a two-page PDF: +- Page 1: Memory Usage and CPU Usage across MPI ranks +- Page 2: Thread Usage with bars for every individual thread + +Usage: + python pa_vis.py [--use-balance-logs] [--use-dat-files] [--use-thread-files] + +By default, tries to use balance logs first, then falls back to .dat files. +""" + +import numpy as np +import sys +import os +import glob +import matplotlib.pyplot as plt +from matplotlib.backends.backend_pdf import PdfPages +import argparse + +def parse_balance_logs(search_dir): + """Parse rank_memory_balance.log and rank_cpu_balance.log files""" + memfile = os.path.join(search_dir, "rank_memory_balance.log") + cpufile = os.path.join(search_dir, "rank_cpu_balance.log") + + if not (os.path.exists(memfile) and os.path.exists(cpufile)): + return None, None + + print(f"Using balance log files: {memfile}, {cpufile}") + + # Load memory data + memdat = np.loadtxt( + memfile, + dtype=[("step", int), ("rank", int), ("resident", float)], + ) + + # Load CPU data + cpudat = np.loadtxt( + cpufile, + dtype=[("step", int), ("rank", int), ("user", float), + ("sys", float), ("sum", float), ("deadfrac", float)], + ) + + return memdat, cpudat + +def parse_thread_info_files(search_dir): + """Parse thread_info files for both serial and MPI runs""" + # Try MPI files first, then serial files + thread_files = glob.glob(os.path.join(search_dir, "thread_info_MPI-step*.dat")) + is_mpi = True + + if not thread_files: + # Try serial pattern + thread_files = glob.glob(os.path.join(search_dir, "thread_info-step*.dat")) + is_mpi = False + + if not thread_files: + print(f"No thread_info files found in {search_dir}") + return None + + print(f"Using {'MPI' if is_mpi else 'serial'} thread info files: {len(thread_files)} files found") + + # Parse thread timing data + thread_data = {} # rank -> {step -> {thread_id -> total_time}} + rank_thread_counts = {} # rank -> max_threads + + for thread_file in sorted(thread_files): + # Extract step from filename: thread_info_MPI-step1.dat + step_part = thread_file.split('-step')[1].split('.dat')[0] + step = int(step_part) + + try: + # Load thread data - format from SWIFT source code analysis + data = np.loadtxt(thread_file) + + if data.size == 0: + continue + + # Handle both single row and multi-row files + if len(data.shape) == 1: + # Single row file + continue + + # Skip first row (metadata) and process task data + task_data = data[1:] if data.shape[0] > 1 else data + + if task_data.size == 0: + continue + + # Process each task line + # Format: rank rid type subtype pair tic toc ci.hydro.count cj.hydro.count ci.grav.count cj.grav.count flags sid + if len(task_data.shape) == 2: + for row in task_data: + if len(row) >= 7: + if is_mpi: + # MPI format: rank rid type subtype pair tic toc ... + rank = int(row[0]) + thread_id = int(row[1]) + tic = int(row[5]) + toc = int(row[6]) + else: + # Serial format: rid type subtype pair tic toc ... + rank = 0 # Serial always uses rank 0 + thread_id = int(row[0]) + tic = int(row[4]) + toc = int(row[5]) + + if toc > tic: # Valid task timing + task_time = toc - tic + + # Initialize data structures + if rank not in thread_data: + thread_data[rank] = {} + rank_thread_counts[rank] = 0 + + if step not in thread_data[rank]: + thread_data[rank][step] = {} + + if thread_id not in thread_data[rank][step]: + thread_data[rank][step][thread_id] = 0 + + thread_data[rank][step][thread_id] += task_time + rank_thread_counts[rank] = max(rank_thread_counts[rank], thread_id + 1) + + except Exception as e: + print(f"Warning: Could not parse {thread_file}: {e}") + continue + + if not thread_data: + return None + + return thread_data, rank_thread_counts + +def parse_dat_files(search_dir): + """Parse memory and communication files for both serial and MPI runs""" + # Try MPI patterns first + memfiles = glob.glob(os.path.join(search_dir, "memuse_report-rank*-step*.dat")) + cpufiles = glob.glob(os.path.join(search_dir, "mpiuse_report-rank*-step*.dat")) + is_mpi = True + + if not memfiles: + # Try serial patterns + memfiles = glob.glob(os.path.join(search_dir, "memuse_report-step*.dat")) + cpufiles = [] # No MPI communication files in serial + is_mpi = False + + if not memfiles: + print(f"No memuse_report files found in {search_dir}") + return None, None + + print(f"Using {'MPI' if is_mpi else 'serial'} .dat files: {len(memfiles)} memuse files, {len(cpufiles)} mpiuse files") + + # Parse memory usage files + mem_data = [] + for memfile in sorted(memfiles): + if is_mpi: + # Extract rank and step from: memuse_report-rank0-step1.dat + parts = os.path.basename(memfile).replace('.dat', '').split('-') + rank = int(parts[1].replace('rank', '')) + step = int(parts[2].replace('step', '')) + else: + # Extract step from: memuse_report-step1.dat + parts = os.path.basename(memfile).replace('.dat', '').split('-') + rank = 0 # Serial always uses rank 0 + step = int(parts[1].replace('step', '')) + + try: + # Load memory data + with open(memfile, 'r') as f: + lines = f.readlines() + + # Find peak memory usage from comments + peak_mem = 0 + for line in lines: + if line.startswith('# Peak memory usage') or 'Memory use by process' in line: + try: + # Extract memory value + parts = line.split() + for i, part in enumerate(parts): + if 'MB' in part: + val = float(parts[i-1]) + peak_mem = val * 1024 # Convert MB to KB + break + elif 'KB' in part: + val = float(parts[i-1]) + peak_mem = val + break + except: + pass + if peak_mem > 0: + break + + if peak_mem == 0: + peak_mem = 1000000 # Default 1GB in KB + + mem_data.append((step, rank, peak_mem)) + + except Exception as e: + print(f"Warning: Could not parse {memfile}: {e}") + continue + + # For serial runs, create dummy CPU data since there's no MPI communication + cpu_data = [] + if not is_mpi and mem_data: + # Create minimal CPU data for serial runs + for step, rank, _ in mem_data: + cpu_data.append((step, rank, 0, 0, 1000, 0)) # Dummy values + elif is_mpi: + # Parse MPI communication files as before + for cpufile in sorted(cpufiles): + # Extract rank and step from filename + parts = os.path.basename(cpufile).replace('.dat', '').split('-') + rank = int(parts[1].replace('rank', '')) + step = int(parts[2].replace('step', '')) + + try: + # Load MPI communication data + with open(cpufile, 'r') as f: + lines = f.readlines() + + # Sum up communication times as a proxy for CPU usage + total_comm_time = 0 + for line in lines: + if line.startswith('#') or not line.strip(): + continue + try: + # Format: stic etic dtic step rank otherrank type itype subtype isubtype activation tag size sum + parts = line.split() + if len(parts) >= 3: + dtic = float(parts[2]) # Duration of MPI operation + total_comm_time += dtic + except: + continue + + # Convert ticks to milliseconds (approximate) + cpu_time_ms = total_comm_time / 1000.0 # Rough conversion + cpu_data.append((step, rank, 0, 0, cpu_time_ms, 0)) # user, sys, sum, deadfrac + + except Exception as e: + print(f"Warning: Could not parse {cpufile}: {e}") + continue + + # Convert to numpy arrays + if mem_data: + memdat = np.array(mem_data, dtype=[("step", int), ("rank", int), ("resident", float)]) + else: + memdat = None + + if cpu_data: + cpudat = np.array(cpu_data, dtype=[ + ("step", int), ("rank", int), ("user", float), + ("sys", float), ("sum", float), ("deadfrac", float) + ]) + else: + cpudat = None + + return memdat, cpudat + +def get_node_assignments(num_ranks): + """Automatically assign node categories based on number of ranks""" + if num_ranks <= 8: + return ["background"] * num_ranks + else: + mid_point = num_ranks // 2 + return ["background"] * mid_point + ["zoom"] * (num_ranks - mid_point) + +def extract_run_info_from_directory(search_dir): + """Extract run information from directory name for PDF naming""" + import re + + dir_name = os.path.basename(search_dir) + + # Try to parse patterns like "data-rivers-N-R-C" where N=nodes, R=ranks, C=cores + patterns = [ + r'data-.*-(\d+)-(\d+)-(\d+)(?:-cosma)?$', # data-rivers-1-2-8-cosma or data-rivers-1-2-8 + r'.*-(\d+)nodes?-(\d+)ranks?-(\d+)cores?', # flexible naming + r'.*-N(\d+)-R(\d+)-C(\d+)', # explicit N-R-C format + r'.*-(\d+)-(\d+)-(\d+)$', # generic three numbers + ] + + for pattern in patterns: + match = re.search(pattern, dir_name) + if match: + nodes, ranks, cores = map(int, match.groups()) + # Check if directory name contains "cosma" (legacy partition indicator) + if 'cosma' in dir_name and not 'cosma5' in dir_name: + return f"{nodes}-{ranks}-{cores}-cosma", nodes # Return both filename and node count + else: + return f"{nodes}-{ranks}-{cores}", nodes + + # Fallback: use directory name if no pattern matches + return dir_name, None + +def detect_partition_from_directory(search_dir): + """Detect SLURM partition from directory name""" + dir_name = os.path.basename(search_dir) + + # Check directory name for partition indicators + if 'cosma' in dir_name.lower() and 'cosma5' not in dir_name.lower(): + return 'cosma' + else: + # Default to cosma5 if no "cosma" suffix or if it's a different pattern + return 'cosma5' + +def format_title_with_partition(base_title, num_steps, num_ranks, num_nodes, partition): + """Format title with partition information""" + if num_nodes is not None: + middle_part = f"{num_steps} selected steps across {num_ranks} MPI ranks on {num_nodes} nodes" + else: + middle_part = f"{num_steps} selected steps across {num_ranks} MPI ranks" + + partition_suffix = f" on {partition}" + + return f"{base_title}\n{middle_part}{partition_suffix}" + +def main(): + parser = argparse.ArgumentParser(description='Performance Analysis Visualization') + parser.add_argument('--use-balance-logs', action='store_true', + help='Force use of rank_*_balance.log files') + parser.add_argument('--use-dat-files', action='store_true', + help='Force use of *_report-rank*-step*.dat files') + parser.add_argument('--use-thread-files', action='store_true', + help='Force use of thread_info_MPI-step*.dat files') + parser.add_argument('--directory', '-d', type=str, default='.', + help='Directory to search for log and dat files (default: current directory)') + args = parser.parse_args() + + # Set the working directory for file searches + search_dir = os.path.abspath(args.directory) + if not os.path.exists(search_dir): + print(f"Error: Directory '{search_dir}' does not exist!") + sys.exit(1) + + # Detect partition from directory name + partition = detect_partition_from_directory(search_dir) + print(f"Detected partition: {partition}") + + # Extract run info and node count for PDF naming + run_info, num_nodes = extract_run_info_from_directory(search_dir) + pdf_filename = f"pa_vis-{run_info}.pdf" + + print(f"Searching for performance files in: {search_dir}") + if num_nodes is not None: + print(f"Detected {num_nodes} nodes from directory name") + print(f"Output will be saved as: {pdf_filename}") + + # Color mapping + cmap = {"zoom": "tab:red", "background": "tab:blue", "compute": "tab:green"} + + # Try to load data + memdat, cpudat = None, None + thread_data, rank_thread_counts = None, None + + if args.use_thread_files: + thread_result = parse_thread_info_files(search_dir) + if thread_result: + thread_data, rank_thread_counts = thread_result + elif args.use_dat_files: + memdat, cpudat = parse_dat_files(search_dir) + thread_result = parse_thread_info_files(search_dir) + if thread_result: + thread_data, rank_thread_counts = thread_result + elif args.use_balance_logs: + memdat, cpudat = parse_balance_logs(search_dir) + else: + # Try balance logs first, then .dat files, then thread files + memdat, cpudat = parse_balance_logs(search_dir) + if memdat is None or cpudat is None: + print("Balance logs not found, trying .dat files...") + memdat, cpudat = parse_dat_files(search_dir) + + # Always try to get thread data + thread_result = parse_thread_info_files(search_dir) + if thread_result: + thread_data, rank_thread_counts = thread_result + + if memdat is None and cpudat is None and thread_data is None: + print("Error: No suitable data files found!") + print("Looking for:") + print(" - rank_memory_balance.log and rank_cpu_balance.log") + print(" - memuse_report-rank*-step*.dat files") + print(" - mpiuse_report-rank*-step*.dat files") + print(" - thread_info_MPI-step*.dat files") + print(" - thread_info_step*.dat files") + sys.exit(1) + + # Get unique ranks and steps + if memdat is not None: + unique_ranks = np.unique(memdat["rank"]) + memory_steps = np.unique(memdat["step"]) + else: + memory_steps = np.array([]) + + if cpudat is not None: + if memdat is None: + unique_ranks = np.unique(cpudat["rank"]) + cpu_steps = np.unique(cpudat["step"]) + else: + cpu_steps = np.array([]) + + if thread_data is not None: + if memdat is None and cpudat is None: + unique_ranks = np.array(sorted(thread_data.keys())) + all_thread_steps = set() + for rank_steps in thread_data.values(): + all_thread_steps.update(rank_steps.keys()) + thread_steps = np.array(sorted(all_thread_steps)) + else: + thread_steps = np.array([]) + + # Use appropriate step count for each page + balance_steps = np.union1d(memory_steps, cpu_steps) + + num_ranks = len(unique_ranks) + num_balance_steps = len(balance_steps) + num_thread_steps = len(thread_steps) + + print(f"Found {num_ranks} ranks and {num_balance_steps} balance steps") + print(f"Ranks: {unique_ranks}") + print(f"Balance Steps: {balance_steps}") + + # Add information about thread steps + if num_thread_steps > 0: + print(f"Thread Steps: {num_thread_steps} steps found") + if num_thread_steps <= 10: + print(f" Steps: {thread_steps}") + else: + print(f" First 5: {thread_steps[:5]}") + print(f" Last 5: {thread_steps[-5:]}") + else: + print("Thread Steps: None") + + # Set up node assignments + node_assignments = get_node_assignments(num_ranks) + + # Calculate accumulated values per rank + accumulated_memory = np.zeros(num_ranks) + accumulated_cpu = np.zeros(num_ranks) + accumulated_thread_time = np.zeros(num_ranks) + thread_efficiency = np.zeros(num_ranks) + max_memory = np.zeros(num_ranks) + + # Individual thread data for detailed visualization + individual_thread_data = {} # rank -> {thread_id -> total_time} + + for i, rank in enumerate(unique_ranks): + # Memory analysis + if memdat is not None: + rank_mask_mem = memdat["rank"] == rank + rank_memory = memdat["resident"][rank_mask_mem] + if len(rank_memory) > 0: + accumulated_memory[i] = np.mean(rank_memory) # Average memory usage + max_memory[i] = np.max(rank_memory) # Peak memory usage + + # CPU analysis + if cpudat is not None: + rank_mask_cpu = cpudat["rank"] == rank + rank_cpu = cpudat["sum"][rank_mask_cpu] + if len(rank_cpu) > 0: + accumulated_cpu[i] = np.sum(rank_cpu) # Total CPU time + + # Thread analysis + if thread_data is not None and rank in thread_data: + total_thread_time = 0 + max_threads = rank_thread_counts.get(rank, 1) + thread_utilization = 0 + + # Initialize individual thread data for this rank + individual_thread_data[rank] = {} + + for step, thread_times in thread_data[rank].items(): + step_total = sum(thread_times.values()) + total_thread_time += step_total + + # Accumulate individual thread times + for thread_id, thread_time in thread_times.items(): + if thread_id not in individual_thread_data[rank]: + individual_thread_data[rank][thread_id] = 0 + individual_thread_data[rank][thread_id] += thread_time + + # Calculate thread utilization for this step + if thread_times: + active_threads = len(thread_times) + thread_utilization += active_threads / max_threads + + accumulated_thread_time[i] = total_thread_time / 1000000000.0 # Convert ticks to seconds + if len(thread_data[rank]) > 0: + thread_efficiency[i] = thread_utilization / len(thread_data[rank]) # Average thread efficiency + + # Set up plotting labels and colors + bar_labels = [] + bar_colors = [] + for i, node_assignment in enumerate(node_assignments): + if node_assignment in bar_labels: + bar_labels.append(f"_{node_assignment}") + else: + bar_labels.append(node_assignment) + bar_colors.append(cmap[node_assignment]) + + # Create the two-page visualization + print("Creating performance analysis visualization...") + + with PdfPages(pdf_filename) as pdffile: + + # PAGE 1: Memory Usage and CPU Usage + has_memory_data = memdat is not None and np.any(accumulated_memory > 0) + has_cpu_data = cpudat is not None and np.any(accumulated_cpu > 0) + + # Initialize plot_idx before the conditional block to avoid UnboundLocalError + plot_idx = 0 + + if has_memory_data or has_cpu_data: + # Calculate number of plots needed + num_plots = int(has_memory_data) + int(has_cpu_data) + + # Fix: Increase figure width to accommodate both charts properly + fig1, axs1 = plt.subplots(1, 2, figsize=(16, 6)) + + # Update title formatting with partition information + title1 = format_title_with_partition( + "Memory and CPU Performance Analysis", + num_balance_steps, + num_ranks, + num_nodes, + partition + ) + + fig1.suptitle(title1, fontsize=14, ha='center', va='top', y=0.95) + + # Memory usage chart + if has_memory_data: + bars_mem = axs1[plot_idx].bar(unique_ranks, accumulated_memory, color=bar_colors, alpha=0.7) + axs1[plot_idx].axhline(np.mean(accumulated_memory), color="black", ls="dashed", lw=1.5, label="Average") + axs1[plot_idx].axhline(1024**2, color="red", ls="solid", lw=2, label="1 GB threshold") # 1024^2 KB + axs1[plot_idx].set_ylabel("Average Memory Usage [KB]") + axs1[plot_idx].set_xlabel("MPI Rank") + axs1[plot_idx].set_title("Memory Usage (Average over selected steps)") + axs1[plot_idx].legend(loc="lower right", fontsize=9) + axs1[plot_idx].grid(True, alpha=0.3) + + # Add text annotations for memory values + for i, (rank, mem_kb) in enumerate(zip(unique_ranks, accumulated_memory)): + mem_gb = mem_kb / (1024**2) + axs1[plot_idx].text(rank, mem_kb + 0.08*(1024**2), f"{mem_gb:.2f}GB", + ha='center', va='bottom', fontsize=10) + + # Set Y-axis limits to provide more space + max_mem = np.max(accumulated_memory) + axs1[plot_idx].set_ylim(0, max_mem * 1.15) # Add 15% padding at top + axs1[plot_idx].set_xticks(unique_ranks) + plot_idx += 1 + + # CPU usage chart + if has_cpu_data: + bars_cpu = axs1[plot_idx].bar(unique_ranks, accumulated_cpu / 1000, # Convert to seconds + color=bar_colors, alpha=0.7) + axs1[plot_idx].axhline(np.mean(accumulated_cpu) / 1000, color="black", + ls="dashed", lw=1.5, label="Average") + axs1[plot_idx].set_ylabel("Total CPU Time [seconds]") + axs1[plot_idx].set_xlabel("MPI Rank") + axs1[plot_idx].set_title("CPU Usage (Total over selected steps)") + + # Calculate and display efficiency + total_cpu_time = accumulated_cpu.sum() + max_cpu_time = accumulated_cpu.max() + efficiency = total_cpu_time / (num_ranks * max_cpu_time) if max_cpu_time > 0 else 0 + + axs1[plot_idx].text(0.95, 0.95, + f"ε = {100*efficiency:.1f}%\nTotal: {total_cpu_time/1000:.1f}s", + transform=axs1[plot_idx].transAxes, va="top", ha="right", + bbox=dict(boxstyle="round,pad=0.3", facecolor="lightgray", alpha=0.7)) + + axs1[plot_idx].legend(loc="lower right", fontsize=9) + axs1[plot_idx].grid(True, alpha=0.3) + + # Add text annotations for CPU values + for i, (rank, cpu_s) in enumerate(zip(unique_ranks, accumulated_cpu / 1000)): + axs1[plot_idx].text(rank, cpu_s + max(accumulated_cpu)/1000 * 0.05, f"{cpu_s:.1f}", + ha='center', va='bottom', fontsize=10) + + # Set Y-axis limits to provide more space + max_cpu_s = np.max(accumulated_cpu) / 1000 + axs1[plot_idx].set_ylim(0, max_cpu_s * 1.15) # Add 15% padding at top + axs1[plot_idx].set_xticks(unique_ranks) + plot_idx += 1 + + # Hide unused subplot if only one plot + if plot_idx == 1: + axs1[1].set_visible(False) + + # Fix: Adjust spacing for wider figure and better layout + plt.subplots_adjust(top=0.85, bottom=0.15, left=0.08, right=0.95, wspace=0.25) + plt.savefig(pdffile, format="pdf", bbox_inches='tight', dpi=150) + plt.close() + + elif thread_data is not None and individual_thread_data: + # If no memory or CPU data but thread data exists, create a message for page 1 + fig1 = plt.figure(figsize=(10, 6)) + plt.text(0.5, 0.5, "No Memory or CPU usage data available.\nSee next page for Thread Usage data.", + ha='center', va='center', fontsize=16) + plt.axis('off') + plt.savefig(pdffile, format="pdf", bbox_inches='tight', dpi=150) + plt.close() + + # PAGE 2: Individual Thread Usage (separated by rank) + if thread_data is not None and individual_thread_data: + # Determine subplot layout based on number of ranks + num_ranks_with_threads = len(individual_thread_data) + + if num_ranks_with_threads == 1: + fig2, ax2 = plt.subplots(1, 1, figsize=(12, 6)) + axs2 = [ax2] + elif num_ranks_with_threads == 2: + fig2, axs2 = plt.subplots(1, 2, figsize=(16, 6)) + elif num_ranks_with_threads <= 4: + fig2, axs2 = plt.subplots(2, 2, figsize=(16, 10)) + axs2 = axs2.flatten() + else: + # For more than 4 ranks, create a grid layout + ncols = min(3, num_ranks_with_threads) + nrows = (num_ranks_with_threads + ncols - 1) // ncols + fig2, axs2 = plt.subplots(nrows, ncols, figsize=(6*ncols, 5*nrows)) + if nrows > 1: + axs2 = axs2.flatten() + else: + axs2 = [axs2] if ncols == 1 else axs2 + + # For Page 2 title + title2 = format_title_with_partition( + "Individual Thread Usage by Rank", + num_thread_steps, + num_ranks, + num_nodes, + partition + ) + + fig2.suptitle(title2, fontsize=14) + + # Plot thread usage for each rank separately + rank_idx = 0 + global_max_thread_time = 0 + + # First pass: find global maximum for consistent y-axis scaling + for rank in sorted(individual_thread_data.keys()): + for thread_time in individual_thread_data[rank].values(): + thread_time_s = thread_time / 1000000000.0 # Convert to seconds + global_max_thread_time = max(global_max_thread_time, thread_time_s) + + # Second pass: create plots + for rank in sorted(individual_thread_data.keys()): + if rank_idx >= len(axs2): + break + + ax = axs2[rank_idx] + + # Get thread data for this rank + rank_threads = individual_thread_data[rank] + thread_ids = sorted(rank_threads.keys()) + thread_times = [rank_threads[tid] / 1000000000.0 for tid in thread_ids] # Convert to seconds + thread_labels = [f"T{tid}" for tid in thread_ids] + + if thread_times: + # Create bar chart for this rank's threads + x_positions = range(len(thread_ids)) + rank_color = bar_colors[rank] if rank < len(bar_colors) else 'tab:gray' + bars = ax.bar(x_positions, thread_times, color=rank_color, alpha=0.7) + + # Add average line for this rank + rank_avg = np.mean(thread_times) + ax.axhline(rank_avg, color="black", ls="dashed", lw=1.5, label=f"Rank {rank} Avg") + + # Set labels and title + ax.set_ylabel("Thread Time [s]") + ax.set_xlabel("Thread ID") + ax.set_title(f"Rank {rank} Thread Usage") + ax.legend(loc="lower right", fontsize=9) + ax.grid(True, alpha=0.3) + + # Set x-axis labels + ax.set_xticks(x_positions) + ax.set_xticklabels(thread_labels) + + # Add text annotations for thread values + for i, (x_pos, thread_time) in enumerate(zip(x_positions, thread_times)): + if thread_time > 0: # Only label non-zero values + ax.text(x_pos, thread_time + global_max_thread_time * 0.02, f"{thread_time:.1f}", + ha='center', va='bottom', fontsize=8, rotation=90) + + # Calculate thread balance metrics for this rank + rank_total_time = sum(thread_times) + rank_max_time = max(thread_times) if thread_times else 0 + rank_min_time = min(thread_times) if thread_times else 0 + rank_balance_ratio = rank_min_time / rank_max_time if rank_max_time > 0 else 0 + + # Add efficiency information + ax.text(0.98, 0.98, + f"Total: {rank_total_time:.1f} s\n" + f"Balance: {rank_balance_ratio:.2f}\n" + f"Threads: {len(thread_times)}", + transform=ax.transAxes, va="top", ha="right", + bbox=dict(boxstyle="round,pad=0.3", facecolor="lightgreen", alpha=0.7), + fontsize=9) + + # Set consistent Y-axis limits across all subplots + ax.set_ylim(0, global_max_thread_time * 1.15) + else: + # No thread data for this rank + ax.text(0.5, 0.5, f"No thread data\nfor Rank {rank}", + transform=ax.transAxes, ha='center', va='center', + bbox=dict(boxstyle="round,pad=0.3", facecolor="lightgray", alpha=0.7)) + ax.set_title(f"Rank {rank} Thread Usage") + ax.set_xlabel("Thread ID") + ax.set_ylabel("Thread Time [s]") + + rank_idx += 1 + + # Hide unused subplots + for i in range(rank_idx, len(axs2)): + axs2[i].set_visible(False) + + plt.tight_layout() + plt.savefig(pdffile, format="pdf", bbox_inches='tight', dpi=150) + plt.close() + + # Print summary statistics + print("\n" + "="*60) + print("PERFORMANCE ANALYSIS SUMMARY") + print("="*60) + print(f"Simulation: {num_balance_steps} steps, {num_ranks} MPI ranks") + + # Show full list for balance steps, first/last for thread steps + if num_balance_steps > 0: + balance_steps_list = ", ".join(map(str, balance_steps)) + print(f"Memory & CPU steps analyzed: {balance_steps_list}") + else: + print("Memory & CPU steps analyzed: None") + + if num_thread_steps > 0: + if num_thread_steps <= 10: + # Show all steps if not too many + thread_steps_list = ", ".join(map(str, thread_steps)) + print(f"Thread steps analyzed: {thread_steps_list}") + else: + # Show first and last few steps + print(f"Thread steps analyzed: {thread_steps[0]} to {thread_steps[-1]} ({num_thread_steps} total steps)") + print(f" First 5: {', '.join(map(str, thread_steps[:5]))}") + print(f" Last 5: {', '.join(map(str, thread_steps[-5:]))}") + else: + print("Thread steps analyzed: None") + + print() + + if memdat is not None and np.any(accumulated_memory > 0): + print("MEMORY USAGE:") + for i, rank in enumerate(unique_ranks): + print(f" Rank {rank}: Avg = {accumulated_memory[i]/1024:.2f} GB, " + f"Max = {max_memory[i]/1024:.2f} GB") + print(f" Overall average: {np.mean(accumulated_memory)/1024:.2f} GB") + print(f" Memory balance: {np.std(accumulated_memory)/np.mean(accumulated_memory):.1%}") + else: + print("MEMORY USAGE: No data available") + + print() + + if cpudat is not None and np.any(accumulated_cpu > 0): + print("CPU USAGE:") + total_cpu_time = accumulated_cpu.sum() + max_cpu_time = accumulated_cpu.max() + efficiency = total_cpu_time / (num_ranks * max_cpu_time) if max_cpu_time > 0 else 0 + + for i, rank in enumerate(unique_ranks): + print(f" Rank {rank}: Total = {accumulated_cpu[i]/1000:.1f} seconds") + print(f" Overall total: {total_cpu_time/1000:.1f} seconds") + print(f" Parallel efficiency: {100*efficiency:.1f}%") + if max_cpu_time > 0: + print(f" Load balance ratio: {accumulated_cpu.min()/max_cpu_time:.2f}") + else: + print("CPU USAGE: No data available") + + print() + + if individual_thread_data: + print("INDIVIDUAL THREAD USAGE:") + total_threads = sum(len(threads) for threads in individual_thread_data.values()) + all_times = [] + for rank, threads in individual_thread_data.items(): + for thread_id, thread_time in threads.items(): + thread_time_ms = thread_time / 1000000000.0 + all_times.append(thread_time_ms) + print(f" Rank {rank} Thread {thread_id}: {thread_time_ms:.1f} s") + + if all_times: + total_thread_time = sum(all_times) + max_thread_time = max(all_times) + min_thread_time = min(all_times) + balance_ratio = min_thread_time / max_thread_time if max_thread_time > 0 else 0 + + print(f" Total threads active: {total_threads}") + print(f" Overall total thread time: {total_thread_time:.1f} s") + print(f" Thread balance ratio: {balance_ratio:.2f}") + else: + print("INDIVIDUAL THREAD USAGE: No data available") + + print() + print(f"Output saved to: {pdf_filename}") + print("="*60) + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/examples/HumanMobility/plot_rivers.py b/examples/HumanMobility/plot_rivers.py new file mode 100644 index 000000000..5ba1b94fd --- /dev/null +++ b/examples/HumanMobility/plot_rivers.py @@ -0,0 +1,64 @@ +import matplotlib + +matplotlib.use("Agg") +import matplotlib.pyplot as plt +import numpy as np +import h5py +import sys +import os + +plt.style.use("../../tools/stylesheets/mnras.mplstyle") + +def plot_rivers(river_filename, output_png, min_x=None, max_x=None, min_y=None, max_y=None): + with h5py.File(river_filename, 'r') as sim: + box_size = sim["/Header"].attrs["BoxSize"] + river_group = sim["/RiverGeometry"] + num_rivers = river_group.attrs.get("num_rivers", 1) + river_geometries = [] + for idx in range(num_rivers): + grp = river_group[f"river_{idx}"] + river_geometries.append({ + "left_bank_x": grp["left_bank_x"][:], + "left_bank_y": grp["left_bank_y"][:], + "right_bank_x": grp["right_bank_x"][:], + "right_bank_y": grp["right_bank_y"][:], + "center_x": grp["centerline_x"][:], + "center_y": grp["centerline_y"][:], + }) + + fig, ax = plt.subplots() + ax.set_title("River Geometry") + + # Plot river banks and centerline + for river in river_geometries: + ax.plot(river["left_bank_x"], river["left_bank_y"], '-', color='b', linewidth=0.8, label=None) + ax.plot(river["right_bank_x"], river["right_bank_y"], '-', color='b', linewidth=0.8, label=None) + ax.plot(river["center_x"], river["center_y"], '--', color='k', linewidth=0.5, label=None) + + if min_x is not None and max_x is not None: + ax.set_xlim(min_x, max_x) + elif "BoxSize" in locals(): + ax.set_xlim(0, box_size[0]) + if min_y is not None and max_y is not None: + ax.set_ylim(min_y, max_y) + elif "BoxSize" in locals(): + ax.set_ylim(0, box_size[1]) + + plt.xlabel("${\\rm{Position}}~x$") + plt.ylabel("${\\rm{Position}}~y$") + plt.savefig(output_png, dpi=300) + plt.close() + +if __name__ == '__main__': + # Usage: python plot_rivers_only.py river_file.hdf5 output.png [min_x max_x min_y max_y] + if len(sys.argv) < 3: + print("Usage: python plot_rivers_only.py river_file.hdf5 output.png [min_x max_x min_y max_y]") + sys.exit(1) + river_filename = sys.argv[1] + output_png = sys.argv[2] + min_x = float(sys.argv[3]) if len(sys.argv) > 3 else None + max_x = float(sys.argv[4]) if len(sys.argv) > 4 else None + min_y = float(sys.argv[5]) if len(sys.argv) > 5 else None + max_y = float(sys.argv[6]) if len(sys.argv) > 6 else None + + plot_rivers(river_filename, output_png, min_x, max_x, min_y, max_y) \ No newline at end of file diff --git a/examples/HumanMobility/plot_velocity_parallel.py b/examples/HumanMobility/plot_velocity_parallel.py index 4ef46ab0b..4ca439a11 100644 --- a/examples/HumanMobility/plot_velocity_parallel.py +++ b/examples/HumanMobility/plot_velocity_parallel.py @@ -12,124 +12,115 @@ import multiprocessing from functools import partial -def process_file(i, min_x, max_x, min_y, max_y): - filename_hdf5 = "data/humanMobility_%04d.hdf5" % i # Adjust the filename pattern as needed - filename_png = "images/humanMobility_%04d.png" % i +def process_file(i, min_x, max_x, min_y, max_y, _type="gas", river_base="river", data_base="data/humanMobility", image_base="images/humanMobility"): + river_filename = f"{river_base}.hdf5" + filename_hdf5 = f"{data_base}_{i:04d}.hdf5" + filename_png = f"{image_base}_{i:04d}.png" + + # Ensure the output directory exists + os.makedirs(os.path.dirname(filename_png), exist_ok=True) + + with h5py.File(river_filename, 'r') as sim: + mass = sim["/Header"].attrs["Mass"] + box_size = sim["/Header"].attrs["BoxSize"] + river_group = sim["/RiverGeometry"] + num_rivers = river_group.attrs.get("num_rivers", 1) + river_geometries = [] + for idx in range(num_rivers): + grp = river_group[f"river_{idx}"] + river_geometries.append({ + "left_bank_x": grp["left_bank_x"][:], + "left_bank_y": grp["left_bank_y"][:], + "right_bank_x": grp["right_bank_x"][:], + "right_bank_y": grp["right_bank_y"][:], + "center_x": grp["centerline_x"][:], + "center_y": grp["centerline_y"][:], + }) - # Check if the PNG file already exists to avoid reprocessing if os.path.exists(filename_png): print(f"File {filename_png} already exists. Skipping.") return # Read data from the HDF5 file with h5py.File(filename_hdf5, 'r') as sim: - - # Read the simulation data - # sim = h5py.File("humanMobility_%04d.hdf5" % snap, "r") boxSize = sim["/Header"].attrs["BoxSize"][0] time = sim["/Header"].attrs["Time"][0] - - # scheme = sim["/HydroScheme"].attrs["Scheme"] - # kernel = sim["/HydroScheme"].attrs["Kernel function"] - # neighbours = sim["/HydroScheme"].attrs["Kernel target N_ngb"] - # eta = sim["/HydroScheme"].attrs["Kernel eta"] - - # point_mass = sim["/Parameters"].attrs["PointMassPotential:mass"] - - mass = sim["/Parameters"].attrs["RiverPotential:mass"] - git = sim["Code"].attrs["Git Revision"] - - pos = sim["/PartType1/Coordinates"][:, :] + part_type = "PartType0" if _type == "gas" else "PartType1" + pos = sim[f"/{part_type}/Coordinates"][:, :] x = pos[:, 0] - boxSize / 2 y = pos[:, 1] - boxSize / 2 - vel = sim["/PartType1/Velocities"][:, :] + vel = sim[f"/{part_type}/Velocities"][:, :] v_norm = np.sqrt(vel[:, 0] ** 2 + vel[:, 1] ** 2) - # rho = sim["/PartType0/Densities"][:] - # u = sim["/PartType0/InternalEnergies"][:] - # S = sim["/PartType0/Entropies"][:] - # P = sim["/PartType0/Pressures"][:] + if _type == "gas": + rho = sim["/PartType0/Densities"][:] + u = sim["/PartType0/InternalEnergies"][:] + S = sim["/PartType0/Entropies"][:] + P = sim["/PartType0/Pressures"][:] X = pos[:, 0] Y = pos[:, 1] U = vel[:, 0] V = vel[:, 1] - - # Plot the interesting quantities - plt.figure(figsize=(7, 7 / 1.6)) - - # Calculate the magnitude of the velocity vectors - # vel_mag = np.sqrt(vel[:, 0]**2 + vel[:, 1]**2) + plt.figure(figsize=(200, 200 / 1.6)) M = np.hypot(V, U) print("Processing %04d, min/max X=(%4.2f, %4.2f), min/max Y (%4.2f, %4.2f), min/max U=(%4.2f, %4.2f), min/max V=(%4.2f, %4.2f)" % \ (i, np.min(X), np.max(X), np.min(Y), np.max(Y), np.min(U), np.max(U), np.min(V), np.max(V))) print("boxsize:", boxSize) - - # Determine a suitable scale for the arrow lengths max_M = np.max(M) - desired_max_arrow_length = 0.02 # Adjust this value as needed + desired_max_arrow_length = 0.02 scale = max_M / desired_max_arrow_length - - # Plotting the velocity vectors using quiver - # Create the plot fig, ax = plt.subplots() ax.set_title("Velocity map of human mobility %04d (scale=%07.2d)\n \"river mass\"=%s" % (i, scale, mass)) + # Plot river banks and centerline + for river in river_geometries: + ax.plot(river["left_bank_x"], river["left_bank_y"], '-', color='0.8', linewidth=0.5, label=None) + ax.plot(river["right_bank_x"], river["right_bank_y"], '-', color='0.8', linewidth=0.5, label=None) + ax.plot(river["center_x"], river["center_y"], '--', color='0.6', linewidth=0.3, label=None) + # Create the quiver plot Q = ax.quiver( - X, # X positions - Y, # Y positions - U, # X components of velocity - V, # Y components of velocity - M, # Color by velocity magnitude - scale=scale, # Scale for arrow lengths - pivot='tip', - # angles='xy', - # scale_units='xy', - # cmap="PuBu", - # width=0.025, - # headwidth=30, - # headlength=50, - # headaxislength=45, - # minshaft=20, - # minlength=1.0 + X, Y, U, V, M, scale=scale, pivot='tip', ) - ax.scatter(X, Y, color='0.5', s=0.2) - - # Add a colorbar to show the magnitude of velocities - # plt.colorbar(label='Velocity Magnitude') - - # Add a colorbar associated with the quiver plot cbar = fig.colorbar(Q, ax=ax, label='Velocity Magnitude') - qk = ax.quiverkey(Q, 0.9, 0.9, 1, r'$1 \frac{m}{s}$', labelpos='E', coordinates='figure') - # Add labels and formatting - # plt.text( - # 0.97, 0.97, "${\\rm{Velocity~vectors}}$", - # ha="right", va="top", transform=plt.gca().transAxes, backgroundcolor="w" - # ) + ax.legend(loc='upper right', fontsize='x-small') plt.xlabel("${\\rm{Position}}~x$", labelpad=0) plt.ylabel("${\\rm{Position}}~y$", labelpad=0) plt.xlim(min_x, max_x) plt.ylim(min_y, max_y) # plt.tight_layout() - - plt.savefig(filename_png, dpi=200) + plt.savefig(filename_png, dpi=600) plt.close() - if __name__ == '__main__': - - # Constants for the arguments - num_files = int(sys.argv[1]) # Adjust to the number of your HDF5 files + # Usage: python plot_velocity_parallel.py num_files min_x max_x min_y max_y type [river_base] [data_base] [image_base] + num_files = int(sys.argv[1]) min_x = int(sys.argv[2]) max_x = int(sys.argv[3]) min_y = int(sys.argv[4]) max_y = int(sys.argv[5]) - - # Create a partial function with fixed additional arguments - partial_process_file = partial(process_file, min_x=min_x, max_x=max_x, min_y=min_y, max_y=max_y) + _type = sys.argv[6] + river_base = sys.argv[7] if len(sys.argv) > 7 else "river" + data_base = sys.argv[8] if len(sys.argv) > 8 else "data/humanMobility" + image_base = sys.argv[9] if len(sys.argv) > 9 else "images/humanMobility" + + if _type not in ["gas", "particles"]: + raise ValueError("type must be either 'gas' or 'particles'") + + partial_process_file = partial( + process_file, + min_x=min_x, + max_x=max_x, + min_y=min_y, + max_y=max_y, + _type=_type, + river_base=river_base, + data_base=data_base, + image_base=image_base, + ) file_indices = list(range(num_files)) diff --git a/examples/HumanMobility/run-flops-dp.sh b/examples/HumanMobility/run-flops-dp.sh new file mode 100755 index 000000000..8fbebd49b --- /dev/null +++ b/examples/HumanMobility/run-flops-dp.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +# Basenames for this run (without extensions) +HUMANS=humans-rivers-3 +RIVERS=river-rivers-3 +HUMANMOBILITY=humanMobility +DATA=data-rivers-3 +IMAGES=images-rivers-3 + +# Render the YAML config from template +export DATA HUMANMOBILITY HUMANS RIVERS +envsubst < humanMobility_template.yml > ${HUMANMOBILITY}.yml + +# Run SWIFT with double-precision FLOPS measurement +likwid-perfctr -f -C 0-15 -g FLOPS_DP -- \ + swift_intel2025 --threads=${SLURM_CPUS_PER_TASK:-16} \ + -A -s -g -G --hm-river --hm-randomwalk -n 1000 ${HUMANMOBILITY}.yml +# likwid-mpirun -np ${SLURM_NTASKS:-1} -t ${SLURM_CPUS_PER_TASK:-16} -omp intel -g FLOPS_DP -- \ +# swift_mpi_intel2025 --threads=${SLURM_CPUS_PER_TASK:-16} \ +# -A -s -g -G --hm-river --hm-randomwalk -n 1000 ${HUMANMOBILITY}.yml diff --git a/examples/HumanMobility/run-mem.sh b/examples/HumanMobility/run-mem.sh new file mode 100755 index 000000000..5a2df35b6 --- /dev/null +++ b/examples/HumanMobility/run-mem.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +# Basenames for this run (without extensions) +HUMANS=humans-rivers-3 +RIVERS=river-rivers-3 +HUMANMOBILITY=humanMobility +DATA=data-rivers-3 +IMAGES=images-rivers-3 + +# Render the YAML config from template +export DATA HUMANMOBILITY HUMANS RIVERS +envsubst < humanMobility_template.yml > ${HUMANMOBILITY}.yml + +# Run SWIFT with memory performance measurement +likwid-perfctr -f -C 0-15 -g MEM -- \ + swift_intel2025 --threads=${SLURM_CPUS_PER_TASK:-16} \ + -A -s -g -G --hm-river --hm-randomwalk -n 1000 ${HUMANMOBILITY}.yml +# likwid-mpirun -np ${SLURM_NTASKS:-1} -t ${SLURM_CPUS_PER_TASK:-16} -omp intel -g MEM -- \ +# swift_mpi_intel2025 --threads=${SLURM_CPUS_PER_TASK:-16} \ +# -A -s -g -G --hm-river --hm-randomwalk -n 1000 ${HUMANMOBILITY}.yml diff --git a/examples/HumanMobility/run.sh b/examples/HumanMobility/run.sh index 5a1f7350e..d25579445 100755 --- a/examples/HumanMobility/run.sh +++ b/examples/HumanMobility/run.sh @@ -1,17 +1,118 @@ - #!/bin/bash -# Remove previously generated data and images to regenerate the new ones -rm humans.hdf5 -rm data/* -#rm images/* +# Default parameters +NUM_HUMANS=1000 + +# Parse command line arguments +while [[ $# -gt 0 ]]; do + case $1 in + --num-humans) + NUM_HUMANS="$2" + shift 2 + ;; + -h|--help) + echo "Usage: $0 [OPTIONS]" + echo "Options:" + echo " --num-humans NUM_HUMANS Number of humans per dimension (default: 1000)" + echo " -h, --help Show this help message" + exit 0 + ;; + *) + echo "Unknown option: $1" + echo "Use -h or --help for usage information" + exit 1 + ;; + esac +done + +echo "Run Configuration:" +echo " Number of humans: ${NUM_HUMANS}x${NUM_HUMANS}" + +# Determine partition suffix based on SLURM partition +partition_suffix="" +if [[ "${SLURM_JOB_PARTITION}" == "cosma" ]]; then + partition_suffix="-cosma" +fi + +# Basenames for this run (without extensions) +HUMANS=humans-rivers-${NUM_HUMANS}${partition_suffix} +RIVERS=river-rivers-${NUM_HUMANS}${partition_suffix} +HUMANMOBILITY=humanMobility + +# Automatically generate directory names based on SLURM parameters +nodes=${SLURM_JOB_NUM_NODES:-1} +ntasks=${SLURM_NTASKS:-1} +cpus_per_task=${SLURM_CPUS_PER_TASK:-16} + +# Generate suffix based on resources and partition +DATA=data-rivers-${nodes}-${ntasks}-${cpus_per_task}${partition_suffix} +IMAGES=images-rivers-${nodes}-${ntasks}-${cpus_per_task}${partition_suffix} + +# Create the data directory if it doesn't exist +if [ -d "${DATA}" ]; then + rm -rf ${DATA}/* +fi +mkdir -p ${DATA} + +# Check if required input files exist +if [ ! -f "${HUMANS}.hdf5" ]; then + echo "ERROR: Input file ${HUMANS}.hdf5 not found!" + echo "Please generate input files first with:" + echo " ./submit.sh --gen -- -n ${NUM_HUMANS}" + exit 1 +fi + +if [ ! -f "${RIVERS}.hdf5" ]; then + echo "ERROR: Input file ${RIVERS}.hdf5 not found!" + echo "Please generate input files first with:" + echo " ./submit.sh --gen -- -n ${NUM_HUMANS}" + exit 1 +fi + +# Render the YAML config from template +export DATA HUMANMOBILITY HUMANS RIVERS +envsubst < humanMobility_template.yml > ${DATA}/${HUMANMOBILITY}.yml + +# Select SWIFT binaries based on partition +if [[ "${SLURM_JOB_PARTITION}" == "cosma" ]]; then + SWIFT=/cosma5/data/durham/dc-niko3/.local/bin/swift_cosma + SWIFT_MPI=/cosma5/data/durham/dc-niko3/.local/bin/swift_mpi_cosma +else + SWIFT=/cosma5/data/durham/dc-niko3/.local/bin/swift_intel2025 + SWIFT_MPI=/cosma5/data/durham/dc-niko3/.local/bin/swift_mpi_intel2025 +fi + +# Enable SWIFT's built-in logging +export SWIFT_TASK_DUMPS=1 +export SWIFT_MPIUSE_REPORTS=1 +export SWIFT_MEMUSE_REPORTS=1 + +echo "Auto-detecting execution mode:" +echo " Partition: ${SLURM_JOB_PARTITION:-cosma5}" +echo " SLURM_JOB_NUM_NODES: $nodes" +echo " SLURM_NTASKS: $ntasks" +echo " SLURM_CPUS_PER_TASK: $cpus_per_task" +echo " Output directory: ${DATA}" +echo " SWIFT binary: $(basename ${SWIFT})" + +# Change to the data directory so all output files are written there +cd ${DATA} -# Generate the initial conditions if they are not present. -if [ ! -e humans.hdf5 ] -then - echo "Generating initial conditions for the human mobility box example..." - python3 makeIC.py -f humans.hdf5 #make_hdf5.py +if [ $ntasks -eq 1 ]; then + echo "Running SERIAL version (1 MPI rank)" + export OMP_NUM_THREADS=$cpus_per_task + + ${SWIFT} --threads=$cpus_per_task \ + --task-dumps=10 -v 1 \ + -A -s -g -G --hm-river --hm-randomwalk -n ${NUM_HUMANS} ${HUMANMOBILITY}.yml +else + echo "Running PARALLEL version ($ntasks MPI ranks)" + export OMP_NUM_THREADS=$cpus_per_task + + mpirun -n $ntasks \ + ${SWIFT_MPI} --threads=$cpus_per_task \ + --task-dumps=10 -v 1 \ + -A -s -g -G --hm-river --hm-randomwalk -n ${NUM_HUMANS} ${HUMANMOBILITY}.yml fi -# Run SWIFT -swift -g --threads=4 -n 10000 humanMobility.yml +echo "Simulation complete. Check output files and SWIFT task logs for analysis in ${DATA}/" \ No newline at end of file diff --git a/examples/HumanMobility/run_perf.sh b/examples/HumanMobility/run_perf.sh new file mode 100755 index 000000000..96c8ce5f8 --- /dev/null +++ b/examples/HumanMobility/run_perf.sh @@ -0,0 +1,365 @@ +#!/bin/bash + +# Unified performance analysis script for SWIFT Human Mobility simulations +# Usage: ./run_perf.sh --TOOL +# where TOOL can be: likwid, maqao, aps, vtune, advisor, inspector, scorep + +# Check if an argument was provided +if [ $# -ne 1 ]; then + echo "Usage: $0 [--likwid|--maqao|--aps|--vtune|--advisor|--inspector|--scorep]" + exit 1 +fi + +mode=$1 + +case $mode in + --likwid|--maqao|--aps|--vtune|--advisor|--inspector|--scorep) + ;; + *) + echo "Invalid argument. Use --likwid, --maqao, --aps, --vtune, --advisor, --inspector, or --scorep" + exit 1 + ;; +esac + +# Extract tool name from argument +TOOL=${mode#--} + +# Basenames for this run (without extensions) +HUMANS=humans-${TOOL} +RIVERS=river-${TOOL} +HUMANMOBILITY=humanMobility + +# Automatically generate directory names based on SLURM parameters +nodes=${SLURM_JOB_NUM_NODES:-1} +ntasks=${SLURM_NTASKS:-1} +cpus_per_task=${SLURM_CPUS_PER_TASK:-16} + +# Determine partition suffix +partition_suffix="" +if [[ "${SLURM_JOB_PARTITION}" == "cosma" ]]; then + partition_suffix="-cosma" +fi + +# Generate suffix based on resources and tool +DATA=data-${TOOL}-${nodes}-${ntasks}-${cpus_per_task}${partition_suffix} +IMAGES=images-${TOOL}-${nodes}-${ntasks}-${cpus_per_task}${partition_suffix} + +# Create the data directory if it doesn't exist +mkdir -p ${DATA} + +# Render the YAML config from template +export DATA HUMANMOBILITY HUMANS RIVERS +envsubst < humanMobility_template.yml > ${DATA}/${HUMANMOBILITY}.yml + +# Select SWIFT binaries based on partition +if [[ "${SLURM_JOB_PARTITION}" == "cosma" ]]; then + SWIFT=/cosma5/data/durham/dc-niko3/.local/bin/swift_cosma + SWIFT_MPI=/cosma5/data/durham/dc-niko3/.local/bin/swift_mpi_cosma +else + SWIFT=/cosma5/data/durham/dc-niko3/.local/bin/swift_intel2025 + SWIFT_MPI=/cosma5/data/durham/dc-niko3/.local/bin/swift_mpi_intel2025 +fi + +echo "${TOOL^^} Analysis Configuration:" +echo " Tool: $TOOL" +echo " Partition: ${SLURM_JOB_PARTITION:-cosma5}" +echo " MPI Tasks: $ntasks" +echo " OpenMP Threads per Task: $cpus_per_task" +echo " Output directory: ${DATA}" + +# Change to the data directory so all output files are written there +cd ${DATA} + +# Enable SWIFT's built-in logging (disable for APS to avoid conflicts) +if [[ "$TOOL" != "aps" ]]; then + export SWIFT_TASK_DUMPS=1 + export SWIFT_MPIUSE_REPORTS=1 + export SWIFT_MEMUSE_REPORTS=1 +fi + +# Tool-specific analysis functions +run_likwid() { + if [ $ntasks -eq 1 ]; then + echo "Running LIKWID on SERIAL SWIFT" + + echo "=== Running LIKWID MEM benchmark ===" + likwid-perfctr -C 0-$((cpus_per_task-1)) -g MEM \ + ${SWIFT} --threads=$cpus_per_task \ + -A -s -g -G --hm-river --hm-randomwalk -n 1000 ${HUMANMOBILITY}.yml --task-dumps=1 \ + > likwid_mem_summary.txt 2>&1 + + echo "=== Running LIKWID FLOPS_DP benchmark ===" + likwid-perfctr -C 0-$((cpus_per_task-1)) -g FLOPS_DP \ + ${SWIFT} --threads=$cpus_per_task \ + -A -s -g -G --hm-river --hm-randomwalk -n 1000 ${HUMANMOBILITY}.yml --task-dumps=1 \ + > likwid_flops_summary.txt 2>&1 + + echo "=== Running LIKWID L3 Cache benchmark ===" + likwid-perfctr -C 0-$((cpus_per_task-1)) -g L3 \ + ${SWIFT} --threads=$cpus_per_task \ + -A -s -g -G --hm-river --hm-randomwalk -n 1000 ${HUMANMOBILITY}.yml --task-dumps=1 \ + > likwid_cache_summary.txt 2>&1 + else + echo "Running LIKWID on PARALLEL SWIFT" + + echo "=== Running LIKWID MEM benchmark (MPI) ===" + srun -n $ntasks -c $cpus_per_task \ + likwid-mpirun -np $ntasks -g MEM \ + ${SWIFT_MPI} --threads=$cpus_per_task \ + -A -s -g -G --hm-river --hm-randomwalk -n 1000 ${HUMANMOBILITY}.yml --task-dumps=1 \ + > likwid_mpi_mem_summary.txt 2>&1 + + echo "=== Running LIKWID FLOPS_DP benchmark (MPI) ===" + srun -n $ntasks -c $cpus_per_task \ + likwid-mpirun -np $ntasks -g FLOPS_DP \ + ${SWIFT_MPI} --threads=$cpus_per_task \ + -A -s -g -G --hm-river --hm-randomwalk -n 1000 ${HUMANMOBILITY}.yml --task-dumps=1 \ + > likwid_mpi_flops_summary.txt 2>&1 + fi + echo "LIKWID analysis complete. Results in likwid_*_summary.txt files" + echo "Check the generated summary files for detailed performance metrics" +} + +run_maqao() { + if [ $ntasks -eq 1 ]; then + echo "Running MAQAO on SERIAL SWIFT" + maqao oneview -R1 --output-format=all \ + --output-dir="maqao_serial_$(date +%Y-%m-%d_%H-%M-%S)" -- \ + ${SWIFT} --threads=$cpus_per_task \ + -A -s -g -G --hm-river --hm-randomwalk -n 1000 ${HUMANMOBILITY}.yml --task-dumps=1 + else + echo "Running MAQAO on PARALLEL SWIFT" + maqao oneview -R1 --output-format=all \ + --mpi-command="srun -n $ntasks -c $cpus_per_task" \ + --envv_OMP_NUM_THREADS="$cpus_per_task" \ + --envv_SWIFT_TASK_DUMPS="1" \ + --envv_SWIFT_MPIUSE_REPORTS="1" \ + --output-dir="maqao_mpi_$(date +%Y-%m-%d_%H-%M-%S)" -- \ + ${SWIFT_MPI} --threads=$cpus_per_task \ + -A -s -g -G --hm-river --hm-randomwalk -n 1000 ${HUMANMOBILITY}.yml --task-dumps=1 + fi + echo "MAQAO analysis complete. Check maqao_*/ directory for results." +} + +run_aps() { + if [ $ntasks -eq 1 ]; then + echo "Running Intel APS on SERIAL SWIFT" + + echo "Running Intel APS performance snapshot analysis..." + aps -r aps_serial_intranode \ + ${SWIFT} --threads=$cpus_per_task \ + -A -s -g -G --hm-river --hm-randomwalk -n 1000 ${HUMANMOBILITY}.yml + + echo "Generating APS summary report..." + aps --report=summary --result-dir=aps_serial_intranode > aps_serial_summary.txt + + echo "Generating APS detailed report..." + aps --report=detailed --result-dir=aps_serial_intranode > aps_serial_detailed.txt + else + echo "Running Intel APS on PARALLEL SWIFT" + + echo "Running Intel APS MPI performance snapshot analysis..." + srun -n $ntasks -c $cpus_per_task \ + aps -r aps_mpi_intranode \ + ${SWIFT_MPI} --threads=$cpus_per_task \ + -A -s -g -G --hm-river --hm-randomwalk -n 1000 ${HUMANMOBILITY}.yml + + echo "Generating APS MPI summary report..." + aps --report=summary --result-dir=aps_mpi_intranode > aps_mpi_summary.txt + + echo "Generating APS MPI detailed report..." + aps --report=detailed --result-dir=aps_mpi_intranode > aps_mpi_detailed.txt + fi + echo "Intel APS analysis complete. Results in aps_*_summary.txt and aps_*_detailed.txt files" + echo "Result directories: aps_*_intranode/" + echo "" + echo "Key APS metrics to check:" + echo " - CPU utilization and efficiency" + echo " - Memory bandwidth utilization" + echo " - Elapsed time and performance characteristics" + echo " - MPI communication overhead (for parallel runs)" + echo "" + echo "To view results:" + echo " cat aps_*_summary.txt # Quick overview" + echo " cat aps_*_detailed.txt # Detailed analysis" +} + +run_vtune() { + if [ $ntasks -eq 1 ]; then + echo "Running VTune on SERIAL SWIFT" + + echo "Running VTune hotspots analysis..." + vtune -collect hotspots -result-dir vtune_hotspots_intranode \ + ${SWIFT} --threads=$cpus_per_task \ + -A -s -g -G --hm-river --hm-randomwalk -n 1000 ${HUMANMOBILITY}.yml --task-dumps=1 + + echo "Generating VTune hotspots summary report..." + vtune -report summary -result-dir vtune_hotspots_intranode > vtune_hotspots_summary.txt + + echo "Running VTune threading analysis..." + vtune -collect threading -result-dir vtune_threading_intranode \ + ${SWIFT} --threads=$cpus_per_task \ + -A -s -g -G --hm-river --hm-randomwalk -n 1000 ${HUMANMOBILITY}.yml --task-dumps=1 + + echo "Generating VTune threading report..." + vtune -report summary -result-dir vtune_threading_intranode > vtune_threading_summary.txt + else + echo "Running VTune on PARALLEL SWIFT" + + echo "Running VTune MPI hotspots analysis..." + srun -n $ntasks -c $cpus_per_task \ + vtune -collect hotspots -result-dir vtune_mpi_hotspots_intranode \ + ${SWIFT_MPI} --threads=$cpus_per_task \ + -A -s -g -G --hm-river --hm-randomwalk -n 1000 ${HUMANMOBILITY}.yml --task-dumps=1 + + echo "Generating VTune MPI summary report..." + vtune -report summary -result-dir vtune_mpi_hotspots_intranode > vtune_mpi_hotspots_summary.txt + fi + echo "VTune analysis complete. Results in vtune_*_summary.txt and vtune_*_intranode/ directories" + echo "To view interactive results, use: vtune-gui vtune_*_intranode" +} + +run_advisor() { + if [ $ntasks -eq 1 ]; then + echo "Running Intel Advisor on SERIAL SWIFT" + + echo "Running Intel Advisor survey analysis..." + advisor --collect=survey --project-dir=advisor_intranode \ + ${SWIFT} --threads=$cpus_per_task \ + -A -s -g -G --hm-river --hm-randomwalk -n 1000 ${HUMANMOBILITY}.yml --task-dumps=1 + + echo "Generating Advisor survey report..." + advisor --report=survey --project-dir=advisor_intranode --format=text > advisor_survey_report.txt + + echo "Running Intel Advisor tripcounts analysis..." + advisor --collect=tripcounts --project-dir=advisor_intranode \ + ${SWIFT} --threads=$cpus_per_task \ + -A -s -g -G --hm-river --hm-randomwalk -n 1000 ${HUMANMOBILITY}.yml --task-dumps=1 + + echo "Generating Advisor tripcounts report..." + advisor --report=tripcounts --project-dir=advisor_intranode --format=text > advisor_tripcounts_report.txt + + echo "Running Intel Advisor map analysis for vectorization opportunities..." + advisor --collect=map --project-dir=advisor_intranode \ + ${SWIFT} --threads=$cpus_per_task \ + -A -s -g -G --hm-river --hm-randomwalk -n 1000 ${HUMANMOBILITY}.yml --task-dumps=1 + + echo "Generating Advisor map report..." + advisor --report=map --project-dir=advisor_intranode --format=text > advisor_map_report.txt + else + echo "Running Intel Advisor on PARALLEL SWIFT" + + echo "Running Intel Advisor MPI survey analysis..." + srun -n $ntasks -c $cpus_per_task \ + advisor --collect=survey --project-dir=advisor_mpi_intranode \ + ${SWIFT_MPI} --threads=$cpus_per_task \ + -A -s -g -G --hm-river --hm-randomwalk -n 1000 ${HUMANMOBILITY}.yml --task-dumps=1 + + echo "Generating Advisor MPI survey report..." + advisor --report=survey --project-dir=advisor_mpi_intranode --format=text > advisor_mpi_survey_report.txt + fi + echo "Intel Advisor analysis complete. Results in advisor_*_report.txt and advisor_*_intranode/ directory" + echo "To view interactive results, use: advisor-gui advisor_*_intranode" +} + +run_inspector() { + if [ $ntasks -eq 1 ]; then + echo "Running Intel Inspector on SERIAL SWIFT" + + echo "Running Intel Inspector memory error analysis..." + inspxe-cl -collect mi2 -result-dir inspector_memory_intranode \ + ${SWIFT} --threads=$cpus_per_task \ + -A -s -g -G --hm-river --hm-randomwalk -n 1000 ${HUMANMOBILITY}.yml --task-dumps=1 + + echo "Generating Inspector memory report..." + inspxe-cl -report summary -result-dir inspector_memory_intranode > inspector_memory_summary.txt + + echo "Running Intel Inspector threading error analysis..." + inspxe-cl -collect ti2 -result-dir inspector_threading_intranode \ + ${SWIFT} --threads=$cpus_per_task \ + -A -s -g -G --hm-river --hm-randomwalk -n 1000 ${HUMANMOBILITY}.yml --task-dumps=1 + + echo "Generating Inspector threading report..." + inspxe-cl -report summary -result-dir inspector_threading_intranode > inspector_threading_summary.txt + else + echo "Running Intel Inspector on PARALLEL SWIFT" + + echo "Running Intel Inspector MPI memory error analysis..." + srun -n $ntasks -c $cpus_per_task \ + inspxe-cl -collect mi2 -result-dir inspector_mpi_memory_intranode \ + ${SWIFT_MPI} --threads=$cpus_per_task \ + -A -s -g -G --hm-river --hm-randomwalk -n 1000 ${HUMANMOBILITY}.yml --task-dumps=1 + + echo "Generating Inspector MPI memory report..." + inspxe-cl -report summary -result-dir inspector_mpi_memory_intranode > inspector_mpi_memory_summary.txt + fi + echo "Intel Inspector analysis complete. Results in inspector_*_summary.txt and inspector_*_intranode/ directories" + echo "To view interactive results, use: inspxe-gui inspector_*_intranode" +} + +run_scorep() { + echo "Running Score-P analysis..." + + # Set Score-P environment variables + export SCOREP_ENABLE_PROFILING=true + export SCOREP_ENABLE_TRACING=false + export SCOREP_PROFILING_MAX_CALLPATH_DEPTH=30 + export SCOREP_TOTAL_MEMORY=1G + + # Note: For Score-P to work properly, SWIFT should be compiled with Score-P instrumentation + echo "Warning: For full Score-P analysis, SWIFT should be recompiled with Score-P instrumentation" + echo "Running with runtime instrumentation only..." + + if [ $ntasks -eq 1 ]; then + echo "Running Score-P on SERIAL SWIFT" + + ${SWIFT} --threads=$cpus_per_task \ + -A -s -g -G --hm-river --hm-randomwalk -n 1000 ${HUMANMOBILITY}.yml --task-dumps=1 + else + echo "Running Score-P on PARALLEL SWIFT" + + srun -n $ntasks -c $cpus_per_task \ + ${SWIFT_MPI} --threads=$cpus_per_task \ + -A -s -g -G --hm-river --hm-randomwalk -n 1000 ${HUMANMOBILITY}.yml --task-dumps=1 + fi + + # Generate Score-P report if profile data exists + if ls scorep-* 1> /dev/null 2>&1; then + echo "Generating Score-P summary report..." + scorep-score scorep-*/profile.cubex > scorep_intranode_summary.txt + + echo "Score-P analysis complete. Results in scorep_intranode_summary.txt and scorep-*/ directory" + echo "Use 'cube scorep-*/profile.cubex' for interactive analysis" + else + echo "No Score-P profile data generated. Consider recompiling SWIFT with Score-P instrumentation." + echo "To enable full Score-P analysis, recompile SWIFT with:" + echo " CC='scorep-gcc' CXX='scorep-g++' ./configure [options]" + fi +} + +# Execute the appropriate analysis function +case $TOOL in + likwid) + run_likwid + ;; + maqao) + run_maqao + ;; + aps) + run_aps + ;; + vtune) + run_vtune + ;; + advisor) + run_advisor + ;; + inspector) + run_inspector + ;; + scorep) + run_scorep + ;; +esac + +echo "Check the generated files for detailed performance metrics in ${DATA}/" \ No newline at end of file diff --git a/examples/HumanMobility/submit-perf.sh b/examples/HumanMobility/submit-perf.sh new file mode 100755 index 000000000..9e9c5ace4 --- /dev/null +++ b/examples/HumanMobility/submit-perf.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +sbatch --job-name="hm_likwid" \ + --output="hm_likwid.out" \ + --error="hm_likwid.err" \ + run_likwid.sh \ No newline at end of file diff --git a/examples/HumanMobility/submit.sh b/examples/HumanMobility/submit.sh new file mode 100755 index 000000000..7acea50da --- /dev/null +++ b/examples/HumanMobility/submit.sh @@ -0,0 +1,122 @@ +#!/bin/bash + +# Submit a job to run the Human Mobility example with specified mode and resources +# ./submit.sh --run -N 1 -n 2 -c 64 # Run on COSMA5 (default, you can append "-p cosma5" optionally) +# ./submit.sh --run -N 1 -n 2 -c 64 -p cosma # Run on COSMA (legacy hardware) +# ./submit.sh --gen -b 50000 -g 5000 -n 500 # Generate with custom parameters +# ./submit.sh --vis -d 1-4-16-cosma # Visualize specific run results +# ./submit.sh --map -d 1-4-16-cosma # Generate map with specific suffix +# Note: The script can be extended to other partitions like Cosma7/Cosma8 + +# Default values +nodes=1 +ntasks=1 +cpus_per_task=64 # Default for COSMA5, will be adjusted for COSMA +partition="cosma5" # Default to cosma5, currently implemented for cosma5|cosma +mode="" + +# Parse command line arguments +mode_args=() # Array to store mode-specific arguments +while [[ $# -gt 0 ]]; do + case $1 in + --gen|--vis|--map|--run|--likwid|--maqao|--aps|--vtune|--advisor|--inspector|--scorep) + mode=$1 + shift + ;; + # SLURM-specific options only + -N|--nodes) + nodes="$2" + shift 2 + ;; + -n|--ntasks) + ntasks="$2" + shift 2 + ;; + -c|--cpus-per-task) + cpus_per_task="$2" + shift 2 + ;; + -p|--partition) + partition="$2" + shift 2 + ;; + -h|--help) + echo "Usage: $0 --TYPE [SLURM_OPTIONS] [-- MODE_SPECIFIC_OPTIONS]" + echo "" + echo "Types:" + echo " gen, vis, map, run, likwid, maqao, aps, vtune, advisor, inspector, scorep" + echo "" + echo "SLURM Options:" + echo " -N, --nodes NODES Number of nodes (default: 1)" + echo " -n, --ntasks NTASKS Number of MPI tasks (default: 1)" + echo " -c, --cpus-per-task CPUS CPUs per task (default: 64 for cosma5, 16 for cosma)" + echo " -p, --partition PARTITION Partition: cosma5 (default), cosma" + echo "" + echo "Mode-specific options should be passed after -- or directly to the mode scripts" + echo "" + echo "Examples:" + echo " $0 --run -N 2 -n 4 -c 32" + echo " $0 --gen -- -b 50000 -g 5000 -n 500" + echo " $0 --vis -- -d 1-4-16-cosma" + exit 0 + ;; + --) + # Everything after -- goes to mode-specific script + shift + mode_args=("$@") + break + ;; + *) + # Unknown options go to mode-specific script + mode_args+=("$1") + shift + ;; + esac +done + +if [ -z "$mode" ]; then + echo "Usage: $0 --TYPE [SLURM_OPTIONS] [-- MODE_SPECIFIC_OPTIONS]" + echo "Types: gen, vis, map, run, likwid, maqao, aps, vtune, advisor, inspector, scorep" + exit 1 +fi + +# Validate partition +case $partition in + cosma5|cosma) + ;; + *) + echo "Error: Invalid partition '$partition'. Use 'cosma5' or 'cosma'" + exit 1 + ;; +esac + +# Adjust default cpus_per_task based on partition if not explicitly set +if [[ "$partition" == "cosma" ]] && [[ "$cpus_per_task" == "64" ]]; then + cpus_per_task=16 +fi + +name=${mode#--} # Remove leading -- from mode + +# Calculate memory per node (estimate: 4GB per core) +mem_per_node=$((cpus_per_task * ntasks / nodes * 4))G + +echo "Submitting to partition: $partition" +echo "Using unified job script: job.sh" +echo "Resources: $nodes nodes, $ntasks tasks, $cpus_per_task cores/task" + +# Show mode-specific arguments if any +if [ ${#mode_args[@]} -gt 0 ]; then + echo "Mode-specific arguments: ${mode_args[*]}" +fi + +# Submit job with specified resources and partition +sbatch --job-name="hm-$name" \ + --output="hm-$name-%N-rank%t-$nodes-$ntasks-$cpus_per_task-$partition.out" \ + --error="hm-$name-%N-rank%t-$nodes-$ntasks-$cpus_per_task-$partition.err" \ + --partition="$partition" \ + --nodes=$nodes \ + --ntasks=$ntasks \ + --ntasks-per-node=$((ntasks / nodes)) \ + --cpus-per-task=$cpus_per_task \ + --mem=$mem_per_node \ + job.sh $mode "${mode_args[@]}" \ No newline at end of file diff --git a/examples/HumanMobility/vis.sh b/examples/HumanMobility/vis.sh new file mode 100755 index 000000000..46c9ea733 --- /dev/null +++ b/examples/HumanMobility/vis.sh @@ -0,0 +1,142 @@ +#!/bin/bash + +# Default parameters +dir_suffix="" +num_files=18 +min_x=4000 +max_x=6000 +min_y=4000 +max_y=6000 +type="gas" +NUM_HUMANS=1000 + +# Parse command line arguments +while [[ $# -gt 0 ]]; do + case $1 in + --num-humans) + NUM_HUMANS="$2" + shift 2 + ;; + -d|--dir-suffix) + dir_suffix="$2" + shift 2 + ;; + --num-files) + num_files="$2" + shift 2 + ;; + --min-x) + min_x="$2" + shift 2 + ;; + --max-x) + max_x="$2" + shift 2 + ;; + --min-y) + min_y="$2" + shift 2 + ;; + --max-y) + max_y="$2" + shift 2 + ;; + --type) + type="$2" + shift 2 + ;; + -h|--help) + echo "Usage: $0 [OPTIONS]" + echo "Options:" + echo " --num-humans NUM_HUMANS Number of humans per dimension (default: 1000)" + echo " -d, --dir-suffix Suffix part of directory path to read data from" + echo " --num-files Number of output files to process (default: 18)" + echo " --min-x Minimum x coordinate for visualization (default: 4000)" + echo " --max-x Maximum x coordinate for visualization (default: 6000)" + echo " --min-y Minimum y coordinate for visualization (default: 4000)" + echo " --max-y Maximum y coordinate for visualization (default: 6000)" + echo " --type Visualization type: gas or particles (default: gas)" + echo " -h, --help Show this help message" + exit 0 + ;; + *) + echo "Unknown option: $1" + echo "Use -h or --help for usage information" + exit 1 + ;; + esac +done + +# If no dir_suffix provided, auto-generate from SLURM parameters +if [ -z "$dir_suffix" ]; then + nodes=${SLURM_JOB_NUM_NODES:-1} + ntasks=${SLURM_NTASKS:-1} + cpus_per_task=${SLURM_CPUS_PER_TASK:-16} + + # Determine partition suffix + partition_suffix="" + if [[ "${SLURM_JOB_PARTITION}" == "cosma" ]]; then + partition_suffix="-cosma" + fi + + dir_suffix="${nodes}-${ntasks}-${cpus_per_task}${partition_suffix}" +else + # Need to determine partition suffix for filename generation + partition_suffix="" + if [[ "${SLURM_JOB_PARTITION}" == "cosma" ]] || [[ "$dir_suffix" == *"-cosma" ]]; then + partition_suffix="-cosma" + fi +fi + +# Basenames for this run (without extensions) +HUMANS=humans-rivers-${NUM_HUMANS}${partition_suffix} +RIVERS=river-rivers-${NUM_HUMANS}${partition_suffix} +HUMANMOBILITY=humanMobility +DATA=data-rivers-${dir_suffix} +IMAGES=images-rivers-${dir_suffix} + +echo "Visualization Configuration:" +echo " Directory suffix: ${dir_suffix}" +echo " Data directory: ${DATA}" +echo " Images directory: ${IMAGES}" +echo " Number of files: ${num_files}" +echo " Visualization bounds: x=[${min_x}, ${max_x}], y=[${min_y}, ${max_y}]" +echo " Visualization type: ${type}" + +# Create images directory if it doesn't exist +mkdir -p ${IMAGES} + +# Remove existing images +rm -f ${IMAGES}/* # don't remore if locally + +# Check if data directory exists +if [ ! -d "${DATA}" ]; then + echo "Error: Data directory '${DATA}' not found!" + echo "Make sure you have run a simulation first or specify the correct directory suffix with -d" + exit 1 +fi + +echo "Processing ${num_files} output files..." + +# Use the new configurable parameters +python3 plot_velocity_parallel.py ${num_files} ${min_x} ${max_x} ${min_y} ${max_y} ${type} ${RIVERS} ${DATA}/${HUMANMOBILITY} ${IMAGES}/${HUMANMOBILITY} + +echo "Visualization complete! Images saved to: ${IMAGES}/" + +# Optional: Generate video (commented out by default) +# python3 generate_GIF.py ${num_files} + +# This command sets: +# - A framerate of 20 frames per second. +# - The input pattern to 'humanMobility_%04d.png' starting from 'humanMobility_0000.png'. +# - The total number of frames to 75 (covering humanMobility_0000.png to humanMobility_0074.png). +# - The output video format to H.264 with yuv420p pixel format for broader compatibility. + +# ffmpeg -framerate 100 \ # uncomment this line if locally +# -start_number 0 \ +# -i ${IMAGES}/humanMobility_%04d.png \ +# -frames:v ${num_files} \ +# -y \ +# -c:v libx264 \ +# -pix_fmt yuv420p \ +# ${IMAGES}/video.mp4 diff --git a/examples/HumanMobility/visualise.sh b/examples/HumanMobility/visualise.sh deleted file mode 100755 index 7a72a1747..000000000 --- a/examples/HumanMobility/visualise.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash - -rm images/* - -num_files=501 -min_x=3000 -max_x=7000 -min_y=3000 -max_y=7000 - -# Plot the result -python3 plot_velocity_parallel.py ${num_files} ${min_x} ${max_x} ${min_y} ${max_y} -python3 generate_GIF.py ${num_files} - -# This command sets: -# - A framerate of 20 frames per second. -# - The input pattern to 'humanMobility_%04d.png' starting from 'humanMobility_0000.png'. -# - The total number of frames to 75 (covering humanMobility_0000.png to humanMobility_0074.png). -# - The output video format to H.264 with yuv420p pixel format for broader compatibility. - -ffmpeg -framerate 20 \ - -start_number 0 \ - -i images/humanMobility_%04d.png \ - -frames:v ${num_files} \ - -y \ - -c:v libx264 \ - -pix_fmt yuv420p \ - video.mp4 diff --git a/install_swift.sh b/install_swift.sh index 460a58022..54baf1309 100755 --- a/install_swift.sh +++ b/install_swift.sh @@ -1,7 +1,7 @@ #!/bin/bash # Adapt the following lines to your system -LOCAL_LIBRARY_PATH=/home/dmitry/local/lib +# LOCAL_LIBRARY_PATH=/home/dmitry/local/lib METIS_PATH=/usr/lib/x86_64-linux-gnu PARMETIS_PATH=/usr/lib/x86_64-linux-gnu @@ -15,8 +15,8 @@ make clean echo "#############################" echo "# Running './configure' ... #" echo "#############################" -./configure CC=gcc \ - LDFLAGS=-L${LOCAL_LIBRARY_PATH} \ +# export CFLAGS="-g -pg" #-fsanitize=address +./configure CC=gcc CFLAGS="-g -pg" LDFLAGS="-pg -lm" \ --enable-mpi \ --enable-parallel-hdf5 \ --with-metis=${METIS_PATH} \ @@ -25,7 +25,11 @@ echo "#############################" --with-hydro-dimension=2 \ --with-abm=human-mobility \ --with-ext-potential=human-mobility \ - --with-hm=river + --with-hm=all + + # LDFLAGS="-pg -L${LOCAL_LIBRARY_PATH}" \ + # --with-hm=random-walk + echo "########################################" echo "# Running 'make -j\$(nproc --all)' ... #" diff --git a/install_swift_cosma.sh b/install_swift_cosma.sh new file mode 100755 index 000000000..1dec6118a --- /dev/null +++ b/install_swift_cosma.sh @@ -0,0 +1,136 @@ +#!/bin/bash + +#SBATCH --nodes=1 # Request 1 node +#SBATCH --ntasks=1 # Single task for compilation +#SBATCH --cpus-per-task=16 # Use all cores for parallel make +#SBATCH --mem=64G # Memory for compilation +#SBATCH -J sw-build # Job name +#SBATCH -o sw_cosma.out # Output file +#SBATCH -e sw_cosma.err # Error file +#SBATCH -p cosma # COSMA5 partition +#SBATCH -A durham # Account +#SBATCH -t 0:30:00 # Increased time for compilation + +source ~/swift-env/bin/activate + +# Adapt the following lines to your system +#LOCAL_LIBRARY_PATH=/home/dmitry/local/lib +#METIS_PATH=/usr/lib/x86_64-linux-gnu +#PARMETIS_PATH=/usr/lib/x86_64-linux-gnu + +module purge +module load cosma +module load intel_comp/2024.2.0 #intel_comp/2025.0.1 +module load umf compiler-rt tbb compiler mpi +# module load openmpi/5.0.3 +module load python +module load fftw/3.3.10 +module load gsl +module load parmetis/4.0.3-64bit +module load parallel_hdf5/1.14.4 +module load sundials/5.8.0_c8_single +module load ffmpeg +# module load likwid/5.4.1 + +# Set number of threads for make +export MAKEFLAGS="-j$SLURM_CPUS_PER_TASK" +# export AR=/cosma/local/intel/oneAPI_2025.0.1/compiler/2025.0/bin/compiler/llvm-ar +# export LD=/cosma/local/intel/oneAPI_2025.0.1/compiler/2025.0/bin/compiler/llvm-link +# export RANLIB=/cosma/local/intel/oneAPI_2025.0.1/compiler/2025.0/bin/compiler/llvm-ranlib + +module list + +# The main installation script for SWIFT_ABM + +echo "############################" +echo "# Running 'make clean' ... #" +echo "############################" +make clean + +echo "#############################" +echo "# Running './configure' ... #" +echo "#############################" +#=============================================================================== +# SWIFT Configuration for Human Mobility Analysis +#=============================================================================== +# +# This configuration builds SWIFT with the following features: +# +# DEBUGGING & PROFILING: +# --enable-debug : Enable debug symbols and consistency checks +# --enable-task-debugging : Enable detailed task-level performance analysis +# (generates thread_info_MPI-step*.dat files) +# +# PARALLELIZATION: +# --enable-mpi : MPI parallel support for multi-node execution +# --enable-parallel-hdf5 : Parallel HDF5 I/O for efficient data output +# --with-tbbmalloc : Intel TBB memory allocator for performance +# --with-parmetis : ParMETIS library for domain decomposition +# +# HUMAN MOBILITY MODEL: +# --with-hydro=abm : Agent-based modeling scheme (a placeholder for future ABM schemes, separating it from the hydro scheme) +# --with-hydro-dimension=2 : 2D simulation space +# --with-abm=human-mobility : Human mobility implementation (a placeholder for future ABM implementations) +# --with-ext-potential=human-mobility : External potential for human mobility +# +# HUMAN MOBILITY SCENARIOS (--with-hm options): +# --with-hm=none : No human mobility scenario (default) +# Disables all HM-specific features +# +# --with-hm=all : Enable all human mobility scenarios +# Includes both river and random-walk cases +# Activates HM_CASE_RIVER and HM_CASE_RANDOMWALK +# +# --with-hm=river : River-based human mobility scenario +# Enables river geography +# Activates HM_CASE_RIVER preprocessor definition +# +# --with-hm=random-walk : Random walk human mobility scenario +# Initiate human random walk which consequently is influenced by other humans and geographic features +# Activates HM_CASE_RANDOMWALK preprocessor definition +# +# NOTE: Human mobility scenarios require: +# - --with-abm=human-mobility (mandatory) +# - --with-hydro=abm (mandatory) +# - --with-ext-potential=human-mobility (recommended) +# - --with-hydro-dimension=2 (typical for HM studies) +# +#=============================================================================== +#/configure --prefix=/cosma/home/do009/dc-niko3/.local/ \ +#export CFLAGS="-fsanitize=address -g" +# --program-suffix=_intel2025 \ +./configure --prefix=/cosma5/data/durham/dc-niko3/.local/ \ + --program-suffix=_cosma \ + CFLAGS="-Wno-error=gnu-folding-constant" \ + LDFLAGS= \ + --enable-debug \ + --enable-task-debugging \ + --enable-mpi \ + --enable-parallel-hdf5 \ + --with-tbbmalloc \ + --with-parmetis \ + --with-hydro=abm \ + --with-hydro-dimension=2 \ + --with-abm=human-mobility \ + --with-ext-potential=human-mobility \ + --with-hm=all + +echo "########################################" +echo "# Running 'make -j\$(nproc --all)' ... #" +echo "########################################" +make + +echo "#####################################" +echo "# Running 'ranlib' on libraries ... #" +echo "#####################################" +ranlib src/.libs/libswiftsim.a argparse/.libs/libargparse.a src/.libs/libswiftsim_mpi.a + +echo "############################" +echo "# Running 'make' again ... #" +echo "############################" +make + +echo "###################################" +echo "# Running 'sudo make install' ... #" +echo "###################################" +make install diff --git a/install_swift_intel2025.sh b/install_swift_intel2025.sh new file mode 100755 index 000000000..c61e27cd5 --- /dev/null +++ b/install_swift_intel2025.sh @@ -0,0 +1,135 @@ +#!/bin/bash + +#SBATCH --nodes=1 # Request 1 node +#SBATCH --ntasks=1 # Single task for compilation +#SBATCH --cpus-per-task=32 # Use all cores for parallel make +#SBATCH --mem=64G # Memory for compilation +#SBATCH -J sw-build # Job name +#SBATCH -o sw_intel2025.out # Output file +#SBATCH -e sw_intel2025.err # Error file +#SBATCH -p cosma5 # COSMA5 partition +#SBATCH -A durham # Account +#SBATCH -t 0:30:00 # Increased time for compilation + +source ~/swift-env/bin/activate + +# Adapt the following lines to your system +#LOCAL_LIBRARY_PATH=/home/dmitry/local/lib +#METIS_PATH=/usr/lib/x86_64-linux-gnu +#PARMETIS_PATH=/usr/lib/x86_64-linux-gnu + +module purge +module load cosma +module load intel_comp/2025.0.1 #intel_comp/2024.2.0 +module load umf compiler-rt tbb compiler mpi +# module load openmpi/5.0.3 +module load python +module load fftw/3.3.10 +module load gsl +module load parmetis/4.0.3-64bit +module load parallel_hdf5/1.14.4 +module load sundials/5.8.0_c8_single +module load ffmpeg +# module load likwid/5.4.1 + +# Set number of threads for make +export MAKEFLAGS="-j$SLURM_CPUS_PER_TASK" +export AR=/cosma/local/intel/oneAPI_2025.0.1/compiler/2025.0/bin/compiler/llvm-ar +export LD=/cosma/local/intel/oneAPI_2025.0.1/compiler/2025.0/bin/compiler/llvm-link +export RANLIB=/cosma/local/intel/oneAPI_2025.0.1/compiler/2025.0/bin/compiler/llvm-ranlib + +module list + +# The main installation script for SWIFT_ABM + +echo "############################" +echo "# Running 'make clean' ... #" +echo "############################" +make clean + +echo "#############################" +echo "# Running './configure' ... #" +echo "#############################" +#=============================================================================== +# SWIFT Configuration for Human Mobility Analysis +#=============================================================================== +# +# This configuration builds SWIFT with the following features: +# +# DEBUGGING & PROFILING: +# --enable-debug : Enable debug symbols and consistency checks +# --enable-task-debugging : Enable detailed task-level performance analysis +# (generates thread_info_MPI-step*.dat files) +# +# PARALLELIZATION: +# --enable-mpi : MPI parallel support for multi-node execution +# --enable-parallel-hdf5 : Parallel HDF5 I/O for efficient data output +# --with-tbbmalloc : Intel TBB memory allocator for performance +# --with-parmetis : ParMETIS library for domain decomposition +# +# HUMAN MOBILITY MODEL: +# --with-hydro=abm : Agent-based modeling scheme (a placeholder for future ABM schemes, separating it from the hydro scheme) +# --with-hydro-dimension=2 : 2D simulation space +# --with-abm=human-mobility : Human mobility implementation (a placeholder for future ABM implementations) +# --with-ext-potential=human-mobility : External potential for human mobility +# +# HUMAN MOBILITY SCENARIOS (--with-hm options): +# --with-hm=none : No human mobility scenario (default) +# Disables all HM-specific features +# +# --with-hm=all : Enable all human mobility scenarios +# Includes both river and random-walk cases +# Activates HM_CASE_RIVER and HM_CASE_RANDOMWALK +# +# --with-hm=river : River-based human mobility scenario +# Enables river geography +# Activates HM_CASE_RIVER preprocessor definition +# +# --with-hm=random-walk : Random walk human mobility scenario +# Initiate human random walk which consequently is influenced by other humans and geographic features +# Activates HM_CASE_RANDOMWALK preprocessor definition +# +# NOTE: Human mobility scenarios require: +# - --with-abm=human-mobility (mandatory) +# - --with-hydro=abm (mandatory) +# - --with-ext-potential=human-mobility (recommended) +# - --with-hydro-dimension=2 (typical for HM studies) +# +#=============================================================================== +#/configure --prefix=/cosma/home/do009/dc-niko3/.local/ \ +#export CFLAGS="-fsanitize=address -g" +./configure --prefix=/cosma5/data/durham/dc-niko3/.local/ \ + --program-suffix=_intel2025 \ + CFLAGS="-Wno-error=gnu-folding-constant" \ + LDFLAGS= \ + --enable-debug \ + --enable-task-debugging \ + --enable-mpi \ + --enable-parallel-hdf5 \ + --with-tbbmalloc \ + --with-parmetis \ + --with-hydro=abm \ + --with-hydro-dimension=2 \ + --with-abm=human-mobility \ + --with-ext-potential=human-mobility \ + --with-hm=all + +echo "########################################" +echo "# Running 'make -j\$(nproc --all)' ... #" +echo "########################################" +make + +echo "#####################################" +echo "# Running 'ranlib' on libraries ... #" +echo "#####################################" +ranlib src/.libs/libswiftsim.a argparse/.libs/libargparse.a src/.libs/libswiftsim_mpi.a + +echo "############################" +echo "# Running 'make' again ... #" +echo "############################" +make + +echo "###################################" +echo "# Running 'sudo make install' ... #" +echo "###################################" +make install diff --git a/src/abm/Geography/river/potential.h b/src/abm/Geography/river/potential.h index 5f58da387..124890bbe 100644 --- a/src/abm/Geography/river/potential.h +++ b/src/abm/Geography/river/potential.h @@ -32,6 +32,8 @@ #include "physical_constants.h" #include "space.h" #include "units.h" +#include "abm/HumanMobility/abm_utils.h" +#include "common_io.h" /** * @brief External Potential Properties - River @@ -39,20 +41,24 @@ */ struct external_potential { - /*! Position of the river (horizontal on the map) */ - double y[2]; // the northern and southern bank of the river - /*! Mass */ double mass; - /*! Width */ - // double width; + /*! Cached acceleration field data */ + float *ax; + float *ay; + double box_size[2]; + int grid_size[2]; - /*! Time-step condition pre-factor */ - // float timestep_mult; + /*! River geometry data for multiple rivers */ + int num_rivers; // Number of rivers + float **left_bank_x; // [num_rivers][bank_points[i]] + float **left_bank_y; + float **right_bank_x; + float **right_bank_y; + int *bank_points; // Number of points for each river }; - /** * @brief Computes the time-step due to the acceleration from river * @@ -75,35 +81,63 @@ __attribute__((always_inline)) INLINE static float external_gravity_timestep( * We change acceleration of humans if they go inside the river. * * @param time The current time. - * @param potential The proerties of the external potential. + * @param potential The properties of the external potential (representing a river). * @param phys_const The physical constants in internal units. - * @param g Pointer to the g-particle data. + * @param g Pointer to the g-particle data (representing a human). */ __attribute__((always_inline)) INLINE static void external_gravity_acceleration( double time, const struct external_potential* restrict potential, const struct phys_const* restrict phys_const, struct gpart* restrict g) { - float dy; - - if(g->x[1] < potential->y[0]) - dy = g->x[1] - potential->y[0]; - else if(g->x[1] > potential->y[1]) - dy = g->x[1] - potential->y[1]; - else { - dy = 0; - message("Human is inside the river!!!"); - } - - const float rinv = 1.f / sqrtf(dy * dy); - const float rinv3 = rinv * rinv * rinv; - - // The acceleration must change when the human is in the river - // g->a_grav[0] += potential->mass * dx * rinv3; - g->a_grav[1] += potential->mass * dy * rinv3; - // g->a_grav[2] += potential->mass * dz * rinv3; - - // gravity_add_comoving_potential(g, value); // value ? + double x, y; + double box_size[2]; + int accel_size[2]; + float *accel_x, *accel_y; + int xi, yi; + float rx, ry; + float ax_ll, ay_ll, ax_lr, ay_lr, ax_ul, ay_ul, ax_ur, ay_ur; + float ax, ay; + + // get the location of a human + x = g->x[0]; + y = g->x[1]; + box_size[0] = potential->box_size[0]; + box_size[1] = potential->box_size[1]; + accel_size[0] = potential->grid_size[0]; + accel_size[1] = potential->grid_size[1]; + accel_x = potential->ax; + accel_y = potential->ay; + + // find the nearest indices of the location of a human + find_nearest_indices_2D(x, y, box_size, accel_size, &xi, &yi, &rx, &ry); //+0.5*box_size[0] + + // correct the indices if they are out of the range + if (xi < 0) xi = 0; + if (xi >= accel_size[0]-1) xi = accel_size[0]-2; + if (yi < 0) yi = 0; + if (yi >= accel_size[1]-1) yi = accel_size[1]-2; + // get x- and y-acceleration at the 4 anchor points + ax_ll = accel_x[xi * accel_size[0] + yi]; + ay_ll = accel_y[xi * accel_size[0] + yi]; + ax_lr = accel_x[(xi+1) * accel_size[0] + yi]; + ay_lr = accel_y[(xi+1) * accel_size[0] + yi]; + ax_ul = accel_x[xi * accel_size[0] + (yi+1)]; + ay_ul = accel_y[xi * accel_size[0] + (yi+1)]; + ax_ur = accel_x[(xi+1) * accel_size[0] + (yi+1)]; + ay_ur = accel_y[(xi+1) * accel_size[0] + (yi+1)]; + + // interpolate acceleration for a human at the location + // (be careful of periodic boundary condition if applicable) + bilinear_interpolation(ax_ll, ay_ll, + ax_lr, ay_lr, + ax_ul, ay_ul, + ax_ur, ay_ur, + rx, ry, + &ax, &ay); + + g->a_grav[0] += ax; + g->a_grav[1] += ay; } /** @@ -124,6 +158,104 @@ external_gravity_get_potential_energy( return 0.f; } +/** + * +*/ +static INLINE void geography_read_acceleration_field( + struct swift_params* parameter_file, struct external_potential* potential) { +#if defined(HAVE_HDF5) + + char filename[DESCRIPTION_BUFFER_SIZE]; + parser_get_param_string(parameter_file, "RiverPotential:parameter_file", filename); + + hid_t file_id = H5Fopen(filename, H5F_ACC_RDONLY, H5P_DEFAULT); + if (file_id < 0) error("Unable to open file %s", filename); + + // Read grid size, box size, and mass + hid_t group_id = H5Gopen(file_id, "Header", H5P_DEFAULT); + if (group_id < 0) error("unable to open group Header.\n"); + io_read_array_attribute(group_id, "BoxSize", DOUBLE, potential->box_size, 2); + io_read_array_attribute(group_id, "GridSize", INT, potential->grid_size, 2); + io_read_attribute(group_id, "Mass", DOUBLE, &potential->mass); + hid_t status = H5Gclose(group_id); + if (status < 0) error("error closing group."); + + // Allocate and read acceleration fields + const long long size = potential->grid_size[0] * potential->grid_size[1]; + potential->ax = (float*)malloc(size * sizeof(float)); + potential->ay = (float*)malloc(size * sizeof(float)); + printf("acceleration grid size: %lld\n", size); + + group_id = H5Gopen(file_id, "AccelerationField", H5P_DEFAULT); + if (group_id < 0) error("unable to open group AccelerationField.\n"); + io_read_array_dataset(group_id, "ax", FLOAT, potential->ax, size); + io_read_array_dataset(group_id, "ay", FLOAT, potential->ay, size); + status = H5Gclose(group_id); + if (status < 0) error("error closing group."); + + // Open RiverGeometry group and read number of rivers + group_id = H5Gopen(file_id, "RiverGeometry", H5P_DEFAULT); + if (group_id < 0) error("unable to open group RiverGeometry.\n"); + + // Read num_rivers attribute + int num_rivers = 1; + if (H5Aexists(group_id, "num_rivers") > 0) { + hid_t attr = H5Aopen(group_id, "num_rivers", H5P_DEFAULT); + H5Aread(attr, H5T_NATIVE_INT, &num_rivers); + H5Aclose(attr); + } + potential->num_rivers = num_rivers; + + // Allocate arrays for each river + potential->left_bank_x = (float**)malloc(num_rivers * sizeof(float*)); + potential->left_bank_y = (float**)malloc(num_rivers * sizeof(float*)); + potential->right_bank_x = (float**)malloc(num_rivers * sizeof(float*)); + potential->right_bank_y = (float**)malloc(num_rivers * sizeof(float*)); + potential->bank_points = (int*)malloc(num_rivers * sizeof(int)); + + // Read each river's geometry + for (int i = 0; i < num_rivers; i++) { + char river_name[32]; + snprintf(river_name, sizeof(river_name), "river_%d", i); + hid_t river_group = H5Gopen(group_id, river_name, H5P_DEFAULT); + if (river_group < 0) error("unable to open group %s.\n", river_name); + + // Get size of bank arrays + hid_t dataset = H5Dopen(river_group, "left_bank_x", H5P_DEFAULT); + if (dataset < 0) error("unable to open dataset left_bank_x.\n"); + hid_t space = H5Dget_space(dataset); + hsize_t dims[1]; + H5Sget_simple_extent_dims(space, dims, NULL); + int npoints = dims[0]; + H5Sclose(space); + H5Dclose(dataset); + + potential->bank_points[i] = npoints; + potential->left_bank_x[i] = (float*)malloc(npoints * sizeof(float)); + potential->left_bank_y[i] = (float*)malloc(npoints * sizeof(float)); + potential->right_bank_x[i] = (float*)malloc(npoints * sizeof(float)); + potential->right_bank_y[i] = (float*)malloc(npoints * sizeof(float)); + + io_read_array_dataset(river_group, "left_bank_x", FLOAT, potential->left_bank_x[i], npoints); + io_read_array_dataset(river_group, "left_bank_y", FLOAT, potential->left_bank_y[i], npoints); + io_read_array_dataset(river_group, "right_bank_x", FLOAT, potential->right_bank_x[i], npoints); + io_read_array_dataset(river_group, "right_bank_y", FLOAT, potential->right_bank_y[i], npoints); + + status = H5Gclose(river_group); + if (status < 0) error("error closing river group."); + } + + status = H5Gclose(group_id); + if (status < 0) error("error closing RiverGeometry group."); + + status = H5Fclose(file_id); + if (status < 0) error("error closing file."); + +#else + message("Cannot read the acceleration field without HDF5"); +#endif +} + /** * @brief Initialises the external potential properties in the internal system * of units. @@ -140,14 +272,8 @@ static INLINE void potential_init_backend( const struct unit_system* us, const struct space* s, struct external_potential* potential) { - /* Read in the position of the centre of potential */ - parser_get_param_double_array(parameter_file, "RiverPotential:position", // northern and southern bank coordinatates - 2, potential->y); - - /* Read the other parameters of the model */ - potential->mass = - parser_get_param_double(parameter_file, "RiverPotential:mass"); - + /* Load acceleration field data and mass parameter */ + geography_read_acceleration_field(parameter_file, potential); } /** @@ -161,4 +287,49 @@ static INLINE void potential_print_backend( message("External potential is 'River'."); } +/** + * @brief Cleans up the external potential by freeing allocated memory. + * + * @param potential The external potential to clean up. + */ +static INLINE void potential_cleanup_backend( + struct external_potential* potential) { + +#if defined(HAVE_HDF5) +/* Free acceleration field arrays if they were allocated */ +if (potential->ax != NULL) { + free(potential->ax); + potential->ax = NULL; +} + +if (potential->ay != NULL) { + free(potential->ay); + potential->ay = NULL; +} + +/* Free river geometry arrays if they were allocated */ +if (potential->left_bank_x != NULL) { + for (int i = 0; i < potential->num_rivers; i++) { + if (potential->left_bank_x[i] != NULL) free(potential->left_bank_x[i]); + if (potential->left_bank_y[i] != NULL) free(potential->left_bank_y[i]); + if (potential->right_bank_x[i] != NULL) free(potential->right_bank_x[i]); + if (potential->right_bank_y[i] != NULL) free(potential->right_bank_y[i]); + } + free(potential->left_bank_x); + free(potential->left_bank_y); + free(potential->right_bank_x); + free(potential->right_bank_y); + potential->left_bank_x = NULL; + potential->left_bank_y = NULL; + potential->right_bank_x = NULL; + potential->right_bank_y = NULL; +} + +if (potential->bank_points != NULL) { + free(potential->bank_points); + potential->bank_points = NULL; +} +#endif + +} #endif /* SWIFT_POTENTIAL_RIVER_H */ diff --git a/src/abm/HumanMobility/abm_iact.h b/src/abm/HumanMobility/abm_iact.h index 69d758d50..1df018ff0 100644 --- a/src/abm/HumanMobility/abm_iact.h +++ b/src/abm/HumanMobility/abm_iact.h @@ -324,6 +324,12 @@ __attribute__((always_inline)) INLINE static void runner_iact_force( const float visc_acc_term = 0.5f * visc * (wi_dr * f_ij + wj_dr * f_ji) * r_inv; +#ifdef HM_CASE_RANDOMWALK + // Seed the random number generator with the current time + srand(time(NULL)); + _kick_random_walk(pi, dx, abm_max_rand, abm_div_rand); + _kick_random_walk(pj, dx, abm_max_rand, abm_div_rand); +#else /* SPH acceleration term */ const float sph_acc_term = (P_over_rho2_i * wi_dr + P_over_rho2_j * wj_dr) * r_inv; @@ -335,12 +341,6 @@ __attribute__((always_inline)) INLINE static void runner_iact_force( /* Assemble the acceleration */ const float acc = sph_acc_term + visc_acc_term + adapt_soft_acc_term; -#ifdef HM_CASE_RANDOMWALK - // Seed the random number generator with the current time - srand(time(NULL)); - _kick_random_walk(pi, dx, abm_max_rand, abm_div_rand); - _kick_random_walk(pj, dx, abm_max_rand, abm_div_rand); -#else /* Use the force Luke ! */ pi->a_hydro[0] -= mj * acc * dx[0]; pi->a_hydro[1] -= mj * acc * dx[1]; @@ -413,7 +413,6 @@ __attribute__((always_inline)) INLINE static void runner_iact_nonsym_force( const float rhoi = pi->rho; const float rhoj = pj->rho; const float pressurei = pi->force.pressure; - const float pressurej = pj->force.pressure; /* Get the kernel for hi. */ const float hi_inv = 1.0f / hi; @@ -437,7 +436,6 @@ __attribute__((always_inline)) INLINE static void runner_iact_nonsym_force( /* Compute gradient terms */ const float P_over_rho2_i = pressurei / (rhoi * rhoi) * f_ij; - const float P_over_rho2_j = pressurej / (rhoj * rhoj) * f_ji; /* Compute dv dot r. */ const float dvdr = (pi->v[0] - pj->v[0]) * dx[0] + @@ -466,6 +464,14 @@ __attribute__((always_inline)) INLINE static void runner_iact_nonsym_force( const float visc_acc_term = 0.5f * visc * (wi_dr * f_ij + wj_dr * f_ji) * r_inv; +#ifdef HM_CASE_RANDOMWALK + // Seed the random number generator with the current time + srand(time(NULL)); + _kick_random_walk(pi, dx, abm_max_rand, abm_div_rand); +#else + const float pressurej = pj->force.pressure; + const float P_over_rho2_j = pressurej / (rhoj * rhoj) * f_ji; + /* SPH acceleration term */ const float sph_acc_term = (P_over_rho2_i * wi_dr + P_over_rho2_j * wj_dr) * r_inv; @@ -477,11 +483,6 @@ __attribute__((always_inline)) INLINE static void runner_iact_nonsym_force( /* Assemble the acceleration */ const float acc = sph_acc_term + visc_acc_term + adapt_soft_acc_term; -#ifdef HM_CASE_RANDOMWALK - // Seed the random number generator with the current time - srand(time(NULL)); - _kick_random_walk(pi, dx, abm_max_rand, abm_div_rand); -#else /* Use the force Luke ! */ pi->a_hydro[0] -= mj * acc * dx[0]; pi->a_hydro[1] -= mj * acc * dx[1]; diff --git a/src/abm/HumanMobility/abm_part.h b/src/abm/HumanMobility/abm_part.h new file mode 100644 index 000000000..4e33764f9 --- /dev/null +++ b/src/abm/HumanMobility/abm_part.h @@ -0,0 +1,235 @@ +/******************************************************************************* + * This file is part of SWIFT_ABM. + * Copyright (c) 2025 Dmitry Nikolaenko (dmitry.nikolaenko@durham.ac.uk) + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + ******************************************************************************/ +#ifndef SWIFT_HUMANMOBILITY_ABM_PART_H +#define SWIFT_HUMANMOBILITY_ABM_PART_H + +/** + * @file HumanMobility/abm_part.h + * @brief Minimal conservative implementation of SPH (Particle definition) + * + * The thermal variable is the internal energy (u). Simple constant + * viscosity term with the Balsara (1995) switch. No thermal conduction + * term is implemented. + * + * This corresponds to equations (43), (44), (45), (101), (103) and (104) with + * \f$\beta=3\f$ and \f$\alpha_u=0\f$ of Price, D., Journal of Computational + * Physics, 2012, Volume 231, Issue 3, pp. 759-794. + */ + +#include "adaptive_softening_struct.h" +#include "black_holes_struct.h" +#include "chemistry_struct.h" +#include "cooling_struct.h" +#include "feedback_struct.h" +#include "mhd_struct.h" +#include "particle_splitting_struct.h" +#include "rt_struct.h" +#include "sink_struct.h" +#include "star_formation_struct.h" +#include "timestep_limiter_struct.h" +#include "tracers_struct.h" + +/** + * @brief Particle fields not needed during the SPH loops over neighbours. + * + * This structure contains the particle fields that are not used in the + * density or force loops. Quantities should be used in the kick, drift and + * potentially ghost tasks only. + */ +struct xpart { + + /*! Offset between current position and position at last tree rebuild. */ + float x_diff[3]; + + /*! Offset between the current position and position at the last sort. */ + float x_diff_sort[3]; + + /*! Velocity at the last full step. */ + float v_full[3]; + + /*! Gravitational acceleration at the end of the last step */ + float a_grav[3]; + + /*! Internal energy at the last full step. */ + float u_full; + + /*! Additional data used to record particle splits */ + struct particle_splitting_data split_data; + + /*! Additional data used to record cooling information */ + struct cooling_xpart_data cooling_data; + + /* Additional data used by the tracers */ + struct tracers_xpart_data tracers_data; + + /* Additional data used by the tracers */ + struct star_formation_xpart_data sf_data; + + /*! Additional data used by the feedback */ + struct feedback_xpart_data feedback_data; + + /*! Additional data used by the MHD scheme */ + struct mhd_xpart_data mhd_data; + +} SWIFT_STRUCT_ALIGN; + +/** + * @brief Particle fields for the SPH particles + * + * The density and force substructures are used to contain variables only used + * within the density and force loops over neighbours. All more permanent + * variables should be declared in the main part of the part structure, + */ +struct part { + + /*! Particle unique ID. */ + long long id; + + /*! Pointer to corresponding gravity part. */ + struct gpart* gpart; + + /*! Particle position. */ + double x[3]; + + /*! Particle predicted velocity. */ + float v[3]; + + /*! Particle acceleration. */ + float a_hydro[3]; + + /*! Particle mass. */ + float mass; + + /*! Particle smoothing length. */ + float h; + + /*! Particle internal energy. */ + float u; + + /*! Time derivative of the internal energy. */ + float u_dt; + + /*! Particle density. */ + float rho; + + /* Store density/force specific stuff. */ + union { + + /** + * @brief Structure for the variables only used in the density loop over + * neighbours. + * + * Quantities in this sub-structure should only be accessed in the density + * loop over neighbours and the ghost task. + */ + struct { + + /*! Neighbour number count. */ + float wcount; + + /*! Derivative of the neighbour number with respect to h. */ + float wcount_dh; + + /*! Derivative of density with respect to h */ + float rho_dh; + + /*! Velocity divergence */ + float div_v; + + /*! Velocity curl */ + float rot_v[3]; + + } density; + + /** + * @brief Structure for the variables only used in the force loop over + * neighbours. + * + * Quantities in this sub-structure should only be accessed in the force + * loop over neighbours and the ghost, drift and kick tasks. + */ + struct { + + /*! "Grad h" term */ + float f; + + /*! Particle pressure. */ + float pressure; + + /*! Particle soundspeed. */ + float soundspeed; + + /*! Particle signal velocity */ + float v_sig; + + /*! Time derivative of smoothing length */ + float h_dt; + + /*! Balsara switch */ + float balsara; + + } force; + }; + + /*! Additional data used for adaptive softening */ + struct adaptive_softening_part_data adaptive_softening_data; + + /*! Additional data used by the MHD scheme */ + struct mhd_part_data mhd_data; + + /*! Chemistry information */ + struct chemistry_part_data chemistry_data; + + /*! Cooling information */ + struct cooling_part_data cooling_data; + + /*! Additional data used by the feedback */ + struct feedback_part_data feedback_data; + + /*! Black holes information (e.g. swallowing ID) */ + struct black_holes_part_data black_holes_data; + + /*! Sink information (e.g. swallowing ID) */ + struct sink_part_data sink_data; + + /*! Additional Radiative Transfer Data */ + struct rt_part_data rt_data; + + /*! RT sub-cycling time stepping data */ + struct rt_timestepping_data rt_time_data; + + /*! Time-step length */ + timebin_t time_bin; + + /*! Time-step limiter information */ + struct timestep_limiter_data limiter_data; + +#ifdef SWIFT_DEBUG_CHECKS + + /* Time of the last drift */ + integertime_t ti_drift; + + /* Time of the last kick */ + integertime_t ti_kick; + +#endif + +} SWIFT_STRUCT_ALIGN; + +#endif /* SWIFT_HUMANMOBILITY_ABM_PART_H */ diff --git a/src/abm/HumanMobility/abm_utils.h b/src/abm/HumanMobility/abm_utils.h new file mode 100644 index 000000000..c604a1479 --- /dev/null +++ b/src/abm/HumanMobility/abm_utils.h @@ -0,0 +1,111 @@ +/******************************************************************************* + * This file is part of SWIFT_ABM. + * Copyright (c) 2025 Dmitry Nikolaenko (dmitry.nikolaenko@durham.ac.uk) + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + ******************************************************************************/ +#ifndef SWIFT_HUMANMOBILITY_ABM_UTILS_H +#define SWIFT_HUMANMOBILITY_ABM_UTILS_H + +/* Standard headers */ +#include + +/** + * @brief A utility to set some initialisation parametres. + * + * @param h The human whose parameters are to be set. + */ +// __attribute__((always_inline)) INLINE static void _abm_rand_params( +// struct part *restrict h, const float max_rand, const float div_rand) { + +// h->max_rand = max_rand; +// h->div_rand = div_rand; +// h->sub_rand = max_rand / div_rand / 2; +// } + + +/** + * @brief Search for indices of the nearest grid points for a given position (x, y) + * in the 2D box of `accel_size` + * describing the area of size `box_size` in which the human is located + * + * @param x The x-coordinate of the human location (x, y). + * @param y The y-coordinate of the human location (x, y). + * @param box_size The size of the box in which the human is located. + * @param accel_size The size of the acceleration grid for the box. + * @param xi The lower x-index of the human location (x, y) in the grid. + * @param yi The lower y-index of the human location (x, y) in the grid. + * @param rx The relative anchor `x` of the human location (x, y) in the grid. + * @param ry The relative anchor `y` of the human location (x, y) in the grid. + */ +__attribute__((always_inline)) +INLINE static void find_nearest_indices_2D(double x, double y, + const double* box_size, + const int* accel_size, + int* xi, int* yi, + float *rx, float *ry) { + *xi = (int)floor(x / box_size[0] * (accel_size[0]-1)); + *yi = (int)floor(y / box_size[1] * (accel_size[1]-1)); + *rx = fmod((x / box_size[0]) * (accel_size[0]-1), 1.0f); + *ry = fmod((y / box_size[1]) * (accel_size[1]-1), 1.0f); +} + + + +/** + * @brief Bilinear interpolation of the acceleration grid for a given position (x, y) + * + * @param h The human whose parameters are to be set. + */ +__attribute__((always_inline)) INLINE static void bilinear_interpolation( + const float ax_ll, const float ay_ll, // Lower-left accelerations + const float ax_lr, const float ay_lr, // Lower-right accelerations + const float ax_ul, const float ay_ul, // Upper-left accelerations + const float ax_ur, const float ay_ur, // Upper-right accelerations + const float rx, const float ry, // Relative position (0-1) + float* ax_out, float* ay_out) { // Output accelerations + + /* Weights for the four corners */ + const float w1 = (1.0f - rx) * (1.0f - ry); // Lower-left + const float w2 = rx * (1.0f - ry); // Lower-right + const float w3 = (1.0f - rx) * ry; // Upper-left + const float w4 = rx * ry; // Upper-right + + /* Interpolate x-acceleration */ + *ax_out = ax_ll * w1 + ax_lr * w2 + ax_ul * w3 + ax_ur * w4; + + /* Interpolate y-acceleration */ + *ay_out = ay_ll * w1 + ay_lr * w2 + ay_ul * w3 + ay_ur * w4; +} + + +/** + * @brief A utility to advance random walk of a human. + * + * @param h The human. + */ +__attribute__((always_inline)) INLINE static void _kick_random_walk( + struct part *restrict h, const float dx[3], const float max_rand, const float div_rand) { + + const float sub_rand = max_rand / div_rand / 2; + + /* Change a_hydro (only 2D for humans) */ + h->a_hydro[0] = (fmod((float)rand(), max_rand) / div_rand - sub_rand) * dx[0]; + h->a_hydro[1] = (fmod((float)rand(), max_rand) / div_rand - sub_rand) * dx[1]; + h->a_hydro[2] = 0.0; //((rand()%max_rand)/div_rand - sub_rand) * dx[2]; +} + + +#endif /* SWIFT_HUMANMOBILITY_ABM_UTILS_H */ diff --git a/src/abm/abm_debug.h b/src/abm/abm_debug.h new file mode 100644 index 000000000..7ef8471a4 --- /dev/null +++ b/src/abm/abm_debug.h @@ -0,0 +1,31 @@ +/******************************************************************************* + * This file is part of SWIFT_ABM. + * Copyright (c) 2025 Dmitry Nikolaenko (dmitry.nikolaenko@durham.ac.uk) + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + ******************************************************************************/ +#ifndef SWIFT_ABM_DEBUG_H +#define SWIFT_ABM_DEBUG_H + +/* Import the right ABM debug variant */ +#if defined(NONE_ABM) +#include "./None/abm_debug.h" +#elif defined(HUMANMOBILITY_ABM) +#include "./HumanMobility/abm_debug.h" +#else +#error "Invalid choice of ABM variant" +#endif + +#endif /* SWIFT_ABM_DEBUG_H */ diff --git a/src/abm/geography.h b/src/abm/geography.h new file mode 100644 index 000000000..52ab82492 --- /dev/null +++ b/src/abm/geography.h @@ -0,0 +1,31 @@ +/******************************************************************************* + * This file is part of SWIFT_ABM. + * Copyright (c) 2025 Dmitry Nikolaenko (dmitry.nikolaenko@durham.ac.uk) + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Lesser General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public License + * along with this program. If not, see . + * + ******************************************************************************/ +#ifndef SWIFT_ABM_GEOGRAPHY_H +#define SWIFT_ABM_GEOGRAPHY_H + +/* Import the right geography definition */ +#if defined(HM_CASE_NONE) +#include "./Geography/none/potential.h" +#elif defined(HM_CASE_RIVER) +#include "./Geography/river/potential.h" +#else +#error "Invalid choice of geography variant" +#endif + +#endif /* SWIFT_ABM_GEOGRAPHY_H */ diff --git a/src/engine.c b/src/engine.c index 7a335c957..a4780ed65 100644 --- a/src/engine.c +++ b/src/engine.c @@ -134,7 +134,8 @@ const char *engine_policy_names[] = {"none", "line of sight", "sink", "rt", - "power spectra"}; + "power spectra", + "abm"}; const int engine_default_snapshot_subsample[swift_type_count] = {0}; @@ -3710,7 +3711,13 @@ void engine_recompute_displacement_constraint(struct engine *e) { * @param restart Was this a run that was restarted from check-point files? */ void engine_clean(struct engine *e, const int fof, const int restart) { - /* Start by telling the runners to stop. */ +#ifdef EXTERNAL_POTENTIAL_ABM + printf("Cleaning up engine...\n"); + // const int with_abm = e->policy & engine_policy_abm; + printf("Cleaning up the external potential\n"); + potential_cleanup_backend((struct external_potential*)e->external_potential); +#endif +/* Start by telling the runners to stop. */ e->step_props = engine_step_prop_done; swift_barrier_wait(&e->run_barrier); diff --git a/src/engine.h b/src/engine.h index a9ebcd22e..587dfcaea 100644 --- a/src/engine.h +++ b/src/engine.h @@ -89,8 +89,9 @@ enum engine_policy { engine_policy_sinks = (1 << 25), engine_policy_rt = (1 << 26), engine_policy_power_spectra = (1 << 27), + engine_policy_abm = (1 << 28), }; -#define engine_maxpolicy 28 +#define engine_maxpolicy 29 extern const char *engine_policy_names[engine_maxpolicy + 1]; /** diff --git a/swift.c b/swift.c index eca50ab20..5fb4e519b 100644 --- a/swift.c +++ b/swift.c @@ -196,6 +196,8 @@ int main(int argc, char *argv[]) { int with_eagle = 0; int with_gear = 0; int with_agora = 0; + int with_hm_randomwalk = 0; + int with_hm_river = 0; int with_line_of_sight = 0; int with_rt = 0; int with_power = 0; @@ -300,6 +302,16 @@ int main(int argc, char *argv[]) { "equivalent to --hydro --limiter --sync --self-gravity --stars " "--star-formation --cooling --feedback.", NULL, 0, 0), + OPT_BOOLEAN( + 0, "hm-randomwalk", &with_hm_randomwalk, + "Run with all the options needed for the random walk model. This is " + "equivalent to --hydro --abm.", + NULL, 0, 0), + OPT_BOOLEAN( + 0, "hm-river", &with_hm_river, + "Run with all the options needed for the river model. This is " + "equivalent to --hydro --abm.", + NULL, 0, 0), OPT_GROUP(" Control options:\n"), OPT_BOOLEAN('a', "pin", &with_aff, @@ -413,6 +425,23 @@ int main(int argc, char *argv[]) { with_cooling = 1; with_feedback = 1; } + if (with_hm_randomwalk) { +#ifndef HM_CASE_RANDOMWALK + error("SWIFT was compiled without random walk case support. Recompile with --with-hm=random-walk or --with-hm=all"); +#endif + with_hydro = 1; + with_abm = 1; + with_self_gravity = 1; // Need this for random walk potential? + } + if (with_hm_river) { +#ifndef HM_CASE_RIVER + error("SWIFT was compiled without river case support. Recompile with --with-hm=river or --with-hm=all"); +#endif + with_hydro = 1; + with_abm = 1; + with_external_gravity = 1; // Need this for river potential + // with_self_gravity = 1; + } /* Deal with thread numbers */ if (nr_threads <= 0) @@ -1547,6 +1576,7 @@ int main(int argc, char *argv[]) { if (with_sinks) engine_policies |= engine_policy_sinks; if (with_rt) engine_policies |= engine_policy_rt; if (with_power) engine_policies |= engine_policy_power_spectra; + if (with_abm) engine_policies |= engine_policy_abm; /* Initialize the engine with the space and policies. */ engine_init(&e, &s, params, output_options, N_total[swift_type_gas],