From ae33251e70025315604de1842f54c1d6a670cf06 Mon Sep 17 00:00:00 2001 From: haraldkl Date: Wed, 26 Nov 2025 14:12:36 +0100 Subject: [PATCH 01/32] Create a first draft for a JOSS paper.md See: https://joss.readthedocs.io/en/latest/paper.html and the example in https://joss.readthedocs.io/en/latest/example_paper.html --- paper.md | 56 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 paper.md diff --git a/paper.md b/paper.md new file mode 100644 index 000000000..dc9801995 --- /dev/null +++ b/paper.md @@ -0,0 +1,56 @@ +--- +title: 'Musubi: Octree based Lattice-Boltzmann solver with support for multiple species' +tags: + - Fortran + - fluid dynamics + - Lattice-Boltzmann method + - Maxwell-Stefan equations + - particle transport + - distributed parallelism +authors: + - name: Kannan Masilamani + orcid: 0000-0002-3640-2154 + equal-contrib: true + affiliation: 1 # (Multiple affiliations must be quoted) + - name: Kartik Jain + orcid: 0000-0002-6540-9304 + equal-contrib: true + affiliation: 2 + - name: Harald Klimach + orcid: 0000-0002-6054-5681 + corresponding: true # (This is how to denote the corresponding author) + affiliation: 3 +affiliations: + - name: German Aerospace Center (DLR), Germany + index: 1 + - name: University of Twente, Netherlands + index: 2 +date: 26 November 2025 +bibliography: paper.bib +--- + +# Summary + +Musubi is a multi-level, parallel lattice Boltzmann solver and part of the APES suite. +It is working on an octree mesh that is linearized by a (Morton) space-filling curve and +uses efficient data structures allowing adaptive parallel simulations. + +Musubi offers several collision kernels (BGK, MRT, HRR, Cumulant) and is designed to deal +with huge meshes (billions of lattices) efficiently. + +It is written in Fortran, with language constructs from Fortran 2003. + +# Statement of need + +Highly resolved fluid simulations are an integral part in many scientifc application areas. +Due to the nonlinearity and the large amount of degrees of freedom to consider in these +problems, these simulations require significant computational resources, which typically +are only available in distributed parallel systems. +Musubi implements the lattice Boltzmann method (LBM) with a Message Passing Interface +(MPI) parallelization with a fully distributed handling of the mesh data, avoiding +bottlenecks on individual processors and enabling the scaling of the simulation to +hundreds of thousands of MPI processes. + +# Acknowlegements + +# References From 9618cf6b4226e996457f11349b4f4f1ded467507 Mon Sep 17 00:00:00 2001 From: haraldkl Date: Wed, 26 Nov 2025 14:37:13 +0100 Subject: [PATCH 02/32] Create paper.bib --- paper.bib | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 paper.bib diff --git a/paper.bib b/paper.bib new file mode 100644 index 000000000..e0d86598d --- /dev/null +++ b/paper.bib @@ -0,0 +1,12 @@ +@article{hasert:2013jc, + author = {Hasert, Manuel and Masilamani, Kannan and Zimny, Simon and Klimach, Harald and Qi, Jiaxing and Bernsdorf, J{\"o}rg and Roller, Sabine}, + doi = {10.1016/j.jocs.2013.11.001}, + issn = {1877-7503}, + journal = {Journal of Computational Science}, + month = {sep}, + number = {5}, + pages = {784--794}, + title = {Complex fluid simulations with the parallel tree-based {Lattice} {Boltzmann} solver {Musubi}}, + volume = {5}, + year = {2014} +} From c69a54bd7c1ec8336dc04c92485a7737394b1643 Mon Sep 17 00:00:00 2001 From: haraldkl Date: Wed, 26 Nov 2025 14:57:27 +0100 Subject: [PATCH 03/32] Create draft-pdf.yml --- .github/workflows/draft-pdf.yml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 .github/workflows/draft-pdf.yml diff --git a/.github/workflows/draft-pdf.yml b/.github/workflows/draft-pdf.yml new file mode 100644 index 000000000..1d7dc44e4 --- /dev/null +++ b/.github/workflows/draft-pdf.yml @@ -0,0 +1,24 @@ +name: Draft PDF +on: [push] + +jobs: + paper: + runs-on: ubuntu-latest + name: Paper Draft + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Build draft PDF + uses: openjournals/openjournals-draft-action@master + with: + journal: joss + # This should be the path to the paper within your repo. + paper-path: paper.md + - name: Upload + uses: actions/upload-artifact@v4 + with: + name: paper + # This is the output path where Pandoc will write the compiled + # PDF. Note, this should be the same directory as the input + # paper.md + path: paper.pdf From 583f6da5d280e85a05fb3093dfac3d39e5610f5a Mon Sep 17 00:00:00 2001 From: haraldkl Date: Wed, 26 Nov 2025 15:26:06 +0100 Subject: [PATCH 04/32] Update paper.md --- paper.md | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/paper.md b/paper.md index dc9801995..5cd6ecbc1 100644 --- a/paper.md +++ b/paper.md @@ -10,16 +10,14 @@ tags: authors: - name: Kannan Masilamani orcid: 0000-0002-3640-2154 - equal-contrib: true affiliation: 1 # (Multiple affiliations must be quoted) - name: Kartik Jain orcid: 0000-0002-6540-9304 - equal-contrib: true affiliation: 2 - name: Harald Klimach orcid: 0000-0002-6054-5681 corresponding: true # (This is how to denote the corresponding author) - affiliation: 3 + affiliation: 1 affiliations: - name: German Aerospace Center (DLR), Germany index: 1 @@ -51,6 +49,28 @@ Musubi implements the lattice Boltzmann method (LBM) with a Message Passing Inte bottlenecks on individual processors and enabling the scaling of the simulation to hundreds of thousands of MPI processes. +# The lattice Boltzmann method + +# The Musubi implementation + +Musubi implements the lattice Boltzmann method in the form of kernels that can be +run on individual refinement levels of an octree mesh. +The interpolation and transformation between the involved levels for the local +refinement are separated from the kernel, allowing for an implementation of the +respective methods without encumberment by the interpolation between the different +resolutions. +This method was described in detail in [hasert:2013jc]. + # Acknowlegements +This software has been written by many people over the years. +The individual authors can be found in each file with the respective copyright +statement. +We want to thank all these contributors, especially the main original developer +of Musubi Manuel Hasert. +Not appearing in the list of authors is Sabine Roller, who enabled the development +of this software in the first place and we are very grateful for this possibility. +We thank our fellow contributors to this code basis Jiaxing Qi, Jens Zudrop, +Simon Zimny, Tristan Vlogman and Mengyu Wang and the many students. + # References From 191e7666f9deefcf6a59cdf3d243609c12bb6a42 Mon Sep 17 00:00:00 2001 From: haraldkl Date: Wed, 26 Nov 2025 15:29:24 +0100 Subject: [PATCH 05/32] Use @ in reference --- paper.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paper.md b/paper.md index 5cd6ecbc1..2bfe552a6 100644 --- a/paper.md +++ b/paper.md @@ -59,7 +59,7 @@ The interpolation and transformation between the involved levels for the local refinement are separated from the kernel, allowing for an implementation of the respective methods without encumberment by the interpolation between the different resolutions. -This method was described in detail in [hasert:2013jc]. +This method was described in detail in [@hasert:2013jc]. # Acknowlegements From 7793babd865eb36ad86d383391e8332b67007c74 Mon Sep 17 00:00:00 2001 From: haraldkl Date: Wed, 26 Nov 2025 16:05:26 +0100 Subject: [PATCH 06/32] Add ror identifiers in paper.md --- paper.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/paper.md b/paper.md index 2bfe552a6..2c24f74fe 100644 --- a/paper.md +++ b/paper.md @@ -19,10 +19,12 @@ authors: corresponding: true # (This is how to denote the corresponding author) affiliation: 1 affiliations: - - name: German Aerospace Center (DLR), Germany + - name: German Aerospace Center, DLR, Germany index: 1 + ror: 04bwf3e34 - name: University of Twente, Netherlands index: 2 + ror: 006hf6230 date: 26 November 2025 bibliography: paper.bib --- @@ -71,6 +73,6 @@ of Musubi Manuel Hasert. Not appearing in the list of authors is Sabine Roller, who enabled the development of this software in the first place and we are very grateful for this possibility. We thank our fellow contributors to this code basis Jiaxing Qi, Jens Zudrop, -Simon Zimny, Tristan Vlogman and Mengyu Wang and the many students. +Simon Zimny, Jana Gericke, Tristan Vlogman, Mengyu Wang and many students. # References From de3efb63347c219eec3fa6318e5e7effca96d5c4 Mon Sep 17 00:00:00 2001 From: haraldkl Date: Thu, 27 Nov 2025 17:55:48 +0100 Subject: [PATCH 07/32] Add palabos to paper.bib --- paper.bib | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/paper.bib b/paper.bib index e0d86598d..4eb944a44 100644 --- a/paper.bib +++ b/paper.bib @@ -10,3 +10,12 @@ @article{hasert:2013jc volume = {5}, year = {2014} } + +@article{Palabos2020, + title = "Palabos: Parallel Lattice Boltzmann Solver", + journal = "Computers & Mathematics with Applications", + year = "2020", + issn = "0898-1221", + doi = "https://doi.org/10.1016/j.camwa.2020.03.022", + author = "Jonas Latt and Orestis Malaspinas and Dimitrios Kontaxakis and Andrea Parmigiani and Daniel Lagrava and Federico Brogi and Mohamed Ben Belgacem and Yann Thorimbert and Sébastien Leclaire and Sha Li and Francesco Marson and Jonathan Lemus and Christos Kotsalos and Raphaël Conradin and Christophe Coreixas and Rémy Petkantchin and Franck Raynaud and Joël Beny and Bastien Chopard", +} From 575c1a89a254bf4c79ccce24701c6bbe999ecc6a Mon Sep 17 00:00:00 2001 From: haraldkl Date: Thu, 27 Nov 2025 17:59:12 +0100 Subject: [PATCH 08/32] Add OpenLB to paper.bib --- paper.bib | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/paper.bib b/paper.bib index 4eb944a44..0bf0d8169 100644 --- a/paper.bib +++ b/paper.bib @@ -19,3 +19,17 @@ @article{Palabos2020 doi = "https://doi.org/10.1016/j.camwa.2020.03.022", author = "Jonas Latt and Orestis Malaspinas and Dimitrios Kontaxakis and Andrea Parmigiani and Daniel Lagrava and Federico Brogi and Mohamed Ben Belgacem and Yann Thorimbert and Sébastien Leclaire and Sha Li and Francesco Marson and Jonathan Lemus and Christos Kotsalos and Raphaël Conradin and Christophe Coreixas and Rémy Petkantchin and Franck Raynaud and Joël Beny and Bastien Chopard", } + +@article{olbPaper2021, + author = {Krause, M.J. and Kummerl\"ander, A. and Avis, S.J. and Kusumaatmaja, H. and Dapelo, D. and Klemens, F. and Gaedtke, M. and Hafen, N. and Mink, A. and Trunk, R. and Marquardt, J.E. and Maier, M.L. and Haussmann, M. and Simonis, S.}, + title = {{OpenLB--Open source lattice Boltzmann code}}, + doi = {https://doi.org/10.1016/j.camwa.2020.04.033}, + issn = {0898-1221}, + pages = {258--288}, + url = {http://www.sciencedirect.com/science/article/pii/S0898122120301875}, + volume = {81}, + abstract = {We present the OpenLB package, a C++ library providing a flexible framework for lattice Boltzmann simulations. The code is publicly available and published under GNU GPLv2, which allows for adaption and implementation of additional models. The extensibility benefits from a modular code structure achieved e.g. by utilizing template meta-programming. The package covers various methodical approaches and is applicable to a wide range of transport problems (e.g. fluid, particulate and thermal flows). The built-in processing of the STL file format furthermore allows for the simple setup of simulations in complex geometries. The utilization of MPI as well as OpenMP parallelism enables the user to perform those simulations on large-scale computing clusters. It requires a minimal amount of dependencies and includes several benchmark cases and examples. The package presented here aims at providing an open access platform for both, applicants and developers, from academia as well as industry, which facilitates the extension of previous implementations and results to novel fields of application for lattice Boltzmann methods. OpenLB was tested and validated over several code reviews and publications. This paper summarizes the findings and gives a brief introduction to the underlying concepts as well as the design of the parallel data structure.}, + journal = {Computers \& Mathematics with Applications}, + keywords = {Numerical simulation, Lattice Boltzmann methods, Partial differential equations, Computational fluid dynamics, Transport problems, OpenLB}, + year = {2021}, +} From 8c7a8f2d47df8b89073912b519872c8f8e6a38eb Mon Sep 17 00:00:00 2001 From: haraldkl Date: Thu, 27 Nov 2025 18:02:50 +0100 Subject: [PATCH 09/32] Add waLBerla to paper.bib --- paper.bib | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/paper.bib b/paper.bib index 0bf0d8169..11959005d 100644 --- a/paper.bib +++ b/paper.bib @@ -33,3 +33,18 @@ @article{olbPaper2021 keywords = {Numerical simulation, Lattice Boltzmann methods, Partial differential equations, Computational fluid dynamics, Transport problems, OpenLB}, year = {2021}, } + +@article{BAUER2021478, +title = {waLBerla: A block-structured high-performance framework for multiphysics simulations}, +journal = {Computers & Mathematics with Applications}, +volume = {81}, +pages = {478-501}, +year = {2021}, +note = {Development and Application of Open-source Software for Problems with Numerical PDEs}, +issn = {0898-1221}, +doi = {https://doi.org/10.1016/j.camwa.2020.01.007}, +url = {https://www.sciencedirect.com/science/article/pii/S0898122120300146}, +author = {Martin Bauer and Sebastian Eibl and Christian Godenschwager and Nils Kohl and Michael Kuron and Christoph Rettinger and Florian Schornbaum and Christoph Schwarzmeier and Dominik Thönnes and Harald Köstler and Ulrich Rüde}, +keywords = {High-performance computing, Multiphysics, Lattice Boltzmann, Rigid particle dynamics, Adaptive mesh refinement, Code generation}, +abstract = {Programming current supercomputers efficiently is a challenging task. Multiple levels of parallelism on the core, on the compute node, and between nodes need to be exploited to make full use of the system. Heterogeneous hardware architectures with accelerators further complicate the development process. waLBerla addresses these challenges by providing the user with highly efficient building blocks for developing simulations on block-structured grids. The block-structured domain partitioning is flexible enough to handle complex geometries, while the structured grid within each block allows for highly efficient implementations of stencil-based algorithms. We present several example applications realized with waLBerla, ranging from lattice Boltzmann methods to rigid particle simulations. Most importantly, these methods can be coupled together, enabling multiphysics simulations. The framework uses meta-programming techniques to generate highly efficient code for CPUs and GPUs from a symbolic method formulation.} +} From d64b71838efe711904ef5b3a3c5c4080c861e972 Mon Sep 17 00:00:00 2001 From: haraldkl Date: Thu, 27 Nov 2025 18:46:43 +0100 Subject: [PATCH 10/32] Add theses to paper.bib --- paper.bib | 63 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) diff --git a/paper.bib b/paper.bib index 11959005d..ab4a921ad 100644 --- a/paper.bib +++ b/paper.bib @@ -11,6 +11,69 @@ @article{hasert:2013jc year = {2014} } +@phdthesis{Hasert:2013, + Annote = {Supervisor: Roller, Sabine}, + Author = {Hasert, Manuel}, + School = {RWTH Aachen University}, + Title = {Multi-Scale Lattice Boltzmann Simulations on Distributed Octrees}, + Year = {2013} +} + +@phdthesis{Zimny:2015, + Annote = {Supervisor: Roller, Sabine}, + Author = {Zimny, Simon}, + School = {RWTH Aachen University}, + Title = {Hemodynamic Flow Simulation in Patient Specific Cerebral Aneurysms}, + Year = {2015} +} + +@phdthesis{Zudrop:2015, + Annote = {Supervisor: Roller, Sabine}, + Author = {Zudrop, Jens}, + School = {RWTH Aachen University}, + Title = {Efficient Numerical Methods for Fluid- and Electrodynamics on Massively Parallel Systems}, + Year = {2015} +} + +@phdthesis{Jain:2016, + Annote = {Supervisor: Roller, Sabine}, + Author = {Jain, Kartik}, + School = {Universit{\"a}t Siegen}, + Title = {Transition to Turbulence in Physiological Flows: Direct Numerical Simulation of Hemodynamics in Intracranial Aneurysms and Cerebrospinal Fluid Hydrodynamics in the Spinal Canal}, + Year = {2016} +} + +@phdthesis{Qi:2017, + Annote = {Supervisor: Roller, Sabine}, + Author = {Qi, Jiaxing}, + School = {RWTH Aachen University}, + Title = {Efficient Lattice Boltzmann Simulations on Large Scale High Performance Computing Systems}, + Year = {2017} +} + +@phdthesis{Masilamani:2020, + Annote = {Supervisor: Roller, Sabine}, + Author = {Masilamani, Kannan}, + School = {RWTH Aachen University}, + Title = {Framework for Coupled Simulation of Electrodialysis Processes}, + Year = {2020} +} + +@phdthesis{Vlogman:2025, + title = "Coupled Lattice Boltzmann and discrete element method simulations with applications to liver radioembolization", + author = "Tristan Vlogman", + year = "2025", + month = jul, + day = "11", + doi = "10.3990/1.9789036567046", + isbn = "978-90-365-6703-9", + publisher = "University of Twente", + address = "Netherlands", + type = "PhD Thesis - Research UT, graduation UT", + school = "University of Twente" +} + + @article{Palabos2020, title = "Palabos: Parallel Lattice Boltzmann Solver", journal = "Computers & Mathematics with Applications", From 1c459f8b087211c34118c593c7386c3c42cea1e1 Mon Sep 17 00:00:00 2001 From: haraldkl Date: Thu, 27 Nov 2025 19:01:08 +0100 Subject: [PATCH 11/32] Formatting in paper.bib --- paper.bib | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/paper.bib b/paper.bib index ab4a921ad..c349b3f1e 100644 --- a/paper.bib +++ b/paper.bib @@ -91,23 +91,20 @@ @article{olbPaper2021 pages = {258--288}, url = {http://www.sciencedirect.com/science/article/pii/S0898122120301875}, volume = {81}, - abstract = {We present the OpenLB package, a C++ library providing a flexible framework for lattice Boltzmann simulations. The code is publicly available and published under GNU GPLv2, which allows for adaption and implementation of additional models. The extensibility benefits from a modular code structure achieved e.g. by utilizing template meta-programming. The package covers various methodical approaches and is applicable to a wide range of transport problems (e.g. fluid, particulate and thermal flows). The built-in processing of the STL file format furthermore allows for the simple setup of simulations in complex geometries. The utilization of MPI as well as OpenMP parallelism enables the user to perform those simulations on large-scale computing clusters. It requires a minimal amount of dependencies and includes several benchmark cases and examples. The package presented here aims at providing an open access platform for both, applicants and developers, from academia as well as industry, which facilitates the extension of previous implementations and results to novel fields of application for lattice Boltzmann methods. OpenLB was tested and validated over several code reviews and publications. This paper summarizes the findings and gives a brief introduction to the underlying concepts as well as the design of the parallel data structure.}, journal = {Computers \& Mathematics with Applications}, keywords = {Numerical simulation, Lattice Boltzmann methods, Partial differential equations, Computational fluid dynamics, Transport problems, OpenLB}, year = {2021}, } @article{BAUER2021478, -title = {waLBerla: A block-structured high-performance framework for multiphysics simulations}, -journal = {Computers & Mathematics with Applications}, -volume = {81}, -pages = {478-501}, -year = {2021}, -note = {Development and Application of Open-source Software for Problems with Numerical PDEs}, -issn = {0898-1221}, -doi = {https://doi.org/10.1016/j.camwa.2020.01.007}, -url = {https://www.sciencedirect.com/science/article/pii/S0898122120300146}, -author = {Martin Bauer and Sebastian Eibl and Christian Godenschwager and Nils Kohl and Michael Kuron and Christoph Rettinger and Florian Schornbaum and Christoph Schwarzmeier and Dominik Thönnes and Harald Köstler and Ulrich Rüde}, -keywords = {High-performance computing, Multiphysics, Lattice Boltzmann, Rigid particle dynamics, Adaptive mesh refinement, Code generation}, -abstract = {Programming current supercomputers efficiently is a challenging task. Multiple levels of parallelism on the core, on the compute node, and between nodes need to be exploited to make full use of the system. Heterogeneous hardware architectures with accelerators further complicate the development process. waLBerla addresses these challenges by providing the user with highly efficient building blocks for developing simulations on block-structured grids. The block-structured domain partitioning is flexible enough to handle complex geometries, while the structured grid within each block allows for highly efficient implementations of stencil-based algorithms. We present several example applications realized with waLBerla, ranging from lattice Boltzmann methods to rigid particle simulations. Most importantly, these methods can be coupled together, enabling multiphysics simulations. The framework uses meta-programming techniques to generate highly efficient code for CPUs and GPUs from a symbolic method formulation.} + title = {waLBerla: A block-structured high-performance framework for multiphysics simulations}, + journal = {Computers & Mathematics with Applications}, + volume = {81}, + pages = {478-501}, + year = {2021}, + issn = {0898-1221}, + doi = {https://doi.org/10.1016/j.camwa.2020.01.007}, + url = {https://www.sciencedirect.com/science/article/pii/S0898122120300146}, + author = {Martin Bauer and Sebastian Eibl and Christian Godenschwager and Nils Kohl and Michael Kuron and Christoph Rettinger and Florian Schornbaum and Christoph Schwarzmeier and Dominik Thönnes and Harald Köstler and Ulrich Rüde}, + keywords = {High-performance computing, Multiphysics, Lattice Boltzmann, Rigid particle dynamics, Adaptive mesh refinement, Code generation}, } From b64a4851e3962bd52d1fd439e6f3718e42266a50 Mon Sep 17 00:00:00 2001 From: haraldkl Date: Thu, 27 Nov 2025 19:01:33 +0100 Subject: [PATCH 12/32] Expand explanations in paper.md --- paper.md | 37 +++++++++++++++++++++++++++++++++---- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/paper.md b/paper.md index 2c24f74fe..6c64231ec 100644 --- a/paper.md +++ b/paper.md @@ -35,8 +35,10 @@ Musubi is a multi-level, parallel lattice Boltzmann solver and part of the APES It is working on an octree mesh that is linearized by a (Morton) space-filling curve and uses efficient data structures allowing adaptive parallel simulations. -Musubi offers several collision kernels (BGK, MRT, HRR, Cumulant) and is designed to deal -with huge meshes (billions of lattices) efficiently. +Musubi is designed to deal with huge meshes (billions of lattices) and complex geometries +on large computing systems efficiently. +It can be used for a wide range of application areas from electrodialysis [@Masilamani:2020] +over biomedical problems [@Jain:2016] to aero-acoustic simulations [@Hasert:2013]. It is written in Fortran, with language constructs from Fortran 2003. @@ -53,6 +55,22 @@ hundreds of thousands of MPI processes. # The lattice Boltzmann method +The lattice Boltzmann method utilizes ideas of cellular automata and represents at its +core a basic two step algorithm. +The state of the fluid is represented by particle density functions (PDF) in a discrete +velocity field. +These PDFs reside on the lattices and are exchanged along the discrete velocity directions. +The two steps of the algorithm are now the streaming of the PDF information along +the velocity directions, followed by the so-called collision, computing the new +PDF on each lattice. +This modeling with discrete velocities also allows for a straight forward handling of +complicated wall boundaries, as a simple line intersection with the wall geometry +can be used to accurately model the surface. +Due to these properties the method has gained popularity in the field of computational +fluid dynamics over the last decades. +Other Open Source solvers that utilize this method are for example Palabos [@Palabos2020], +OpenLB [@olbPaper2021] and waLBerla [@BAUER2021478]. + # The Musubi implementation Musubi implements the lattice Boltzmann method in the form of kernels that can be @@ -62,6 +80,10 @@ refinement are separated from the kernel, allowing for an implementation of the respective methods without encumberment by the interpolation between the different resolutions. This method was described in detail in [@hasert:2013jc]. +There are various collision schemes implemented (BGK, MRT, HRR, Cumulants) which can +be used on a range for stencil configurations (discrete velocity directions). +It is also possible to consider the transport of particles and passive scalars in +the flow. # Acknowlegements @@ -72,7 +94,14 @@ We want to thank all these contributors, especially the main original developer of Musubi Manuel Hasert. Not appearing in the list of authors is Sabine Roller, who enabled the development of this software in the first place and we are very grateful for this possibility. -We thank our fellow contributors to this code basis Jiaxing Qi, Jens Zudrop, -Simon Zimny, Jana Gericke, Tristan Vlogman, Mengyu Wang and many students. +We thank our fellow contributors to this code basis Jiaxing Qi [@Qi:2017], +Jens Zudrop [@Zudrop:2015], Simon Zimny [@Zimny:2015], Peter Vitt, Jana Gericke, +Tristan Vlogman [@Vlogman:2025], Mengyu Wang and many students. +The development of Musubi was partially funded by the German Federal Ministry of Education and Research +(Bundesministerium für Bildung und Forschung, BMBF) in the framework of the HPC software initiative in +the project HISEEM and by the European Commission in the Seventh Framework Programme in the area of +Virtual Physiological Human (THROMBUS project, ICT-2009.5.3, project reference 269966). +We are grateful for the computing time provided by LRZ in Munich and by HLRS in Stuttgart who +also contributed performance evaluations within the POP project. # References From 3b3d0285b52ecc808d26407b70f5c13233203f09 Mon Sep 17 00:00:00 2001 From: haraldkl Date: Fri, 28 Nov 2025 10:54:40 +0100 Subject: [PATCH 13/32] Add more references in paper.bib --- paper.bib | 57 +++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 55 insertions(+), 2 deletions(-) diff --git a/paper.bib b/paper.bib index c349b3f1e..358d1a588 100644 --- a/paper.bib +++ b/paper.bib @@ -1,3 +1,11 @@ +@article{Klimach:2012vi, + author = {Klimach, Harald and Hasert, Manuel and Zudrop, Jens and Roller, Sabine}, + title = {{Distributed Octree Mesh Infrastructure for Flow Simulations}}, + journal = {European Congress on Computational Methods in Applied Sciences and Engineering}, + year = {2012}, + pages = {1--15} +} + @article{hasert:2013jc, author = {Hasert, Manuel and Masilamani, Kannan and Zimny, Simon and Klimach, Harald and Qi, Jiaxing and Bernsdorf, J{\"o}rg and Roller, Sabine}, doi = {10.1016/j.jocs.2013.11.001}, @@ -19,6 +27,24 @@ @phdthesis{Hasert:2013 Year = {2013} } +@inproceedings{Klimach:2014, + address = {Munich, Germany}, + doi = {10.3233/978-1-61499-381-0-703}, + author = {Klimach, Harald and Jain, Kartik and Roller, Sabine}, + booktitle = {Parallel Computing: Accelerating Computational Science and Engineering (CSE)}, + date-added = {2014-03-05 14:36:31 +0100}, + date-modified = {2016-04-21 07:20:21 +0000}, + editor = {Bader, Michael and Bode, Arndt and Bungartz, Hans-Joachim and Gerndt, Michael and Joubert, Gerhard R. and Peters, Frans}, + month = {sep}, + pages = {703--711}, + publisher = {IOS Press}, + series = {Advances in Parallel Computing}, + title = {End-to-end parallel simulations with APES}, + volume = {25}, + year = {2014} +} + + @phdthesis{Zimny:2015, Annote = {Supervisor: Roller, Sabine}, Author = {Zimny, Simon}, @@ -93,7 +119,7 @@ @article{olbPaper2021 volume = {81}, journal = {Computers \& Mathematics with Applications}, keywords = {Numerical simulation, Lattice Boltzmann methods, Partial differential equations, Computational fluid dynamics, Transport problems, OpenLB}, - year = {2021}, + year = {2021} } @article{BAUER2021478, @@ -106,5 +132,32 @@ @article{BAUER2021478 doi = {https://doi.org/10.1016/j.camwa.2020.01.007}, url = {https://www.sciencedirect.com/science/article/pii/S0898122120300146}, author = {Martin Bauer and Sebastian Eibl and Christian Godenschwager and Nils Kohl and Michael Kuron and Christoph Rettinger and Florian Schornbaum and Christoph Schwarzmeier and Dominik Thönnes and Harald Köstler and Ulrich Rüde}, - keywords = {High-performance computing, Multiphysics, Lattice Boltzmann, Rigid particle dynamics, Adaptive mesh refinement, Code generation}, + keywords = {High-performance computing, Multiphysics, Lattice Boltzmann, Rigid particle dynamics, Adaptive mesh refinement, Code generation} +} + +@article{Spinelli:2023, + title = {HPC performance study of different collision models using the Lattice Boltzmann solver Musubi}, + journal = {Computers & Fluids}, + volume = {255}, + pages = {105833}, + year = {2023}, + issn = {0045-7930}, + doi = {https://doi.org/10.1016/j.compfluid.2023.105833}, + url = {https://www.sciencedirect.com/science/article/pii/S0045793023000580}, + author = {Gregorio Gerardo Spinelli and Tobias Horstmann and Kannan Masilamani and Malav Mukesh Soni and Harald Klimach and Arthur Stück and Sabine Roller}, + keywords = {Lattice Boltzmann method, Hybrid recursive regularized BGK, Cumulant, Turbulence modeling, Musubi} +} + +@articleInfo{Spinelli:2024, + title = {Key ingredients for wall-modeled LES with the Lattice Boltzmann method: Systematic comparison of collision schemes, SGS models, and wall functions on simulation accuracy and efficiency for turbulent channel flow}, + journal = {Discrete and Continuous Dynamical Systems - S}, + volume = {17}, + number = {11}, + pages = {3224-3251} + year = {2024}, + issn = {1937-1632}, + doi = {10.3934/dcdss.2023212}, + url = {https://www.aimsciences.org/article/id/6578251d4aaea52710702d59}, + author = {Gregorio Gerardo Spinelli and Jana Gericke and Kannan Masilamani and Harald Günther Klimach}, + keywords = {Lattice Boltzmann method, Hybrid recursive regularized BGK, Cumulant, turbulence modeling, Musubi, wall functions} } From e11a6f5a58598dfcbd8f468f75401fe7a63aac33 Mon Sep 17 00:00:00 2001 From: haraldkl Date: Fri, 28 Nov 2025 10:55:18 +0100 Subject: [PATCH 14/32] Expand explanations and mention Treelm in paper.md --- paper.md | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/paper.md b/paper.md index 6c64231ec..ae24ff9cf 100644 --- a/paper.md +++ b/paper.md @@ -1,5 +1,5 @@ --- -title: 'Musubi: Octree based Lattice-Boltzmann solver with support for multiple species' +title: 'Musubi: Octree based Lattice-Boltzmann solver for multi-pyhsics' tags: - Fortran - fluid dynamics @@ -14,6 +14,8 @@ authors: - name: Kartik Jain orcid: 0000-0002-6540-9304 affiliation: 2 + - name: Manuel Hasert + affiliation: 3 - name: Harald Klimach orcid: 0000-0002-6054-5681 corresponding: true # (This is how to denote the corresponding author) @@ -25,7 +27,10 @@ affiliations: - name: University of Twente, Netherlands index: 2 ror: 006hf6230 -date: 26 November 2025 + - name: Festo SE & Co. KG + index: 3 + ror: 03ga8q162 +date: 28 November 2025 bibliography: paper.bib --- @@ -38,7 +43,8 @@ uses efficient data structures allowing adaptive parallel simulations. Musubi is designed to deal with huge meshes (billions of lattices) and complex geometries on large computing systems efficiently. It can be used for a wide range of application areas from electrodialysis [@Masilamani:2020] -over biomedical problems [@Jain:2016] to aero-acoustic simulations [@Hasert:2013]. +over biomedical problems [@Jain:2016] and aero-dynamic setups [@Spinellis:2024] to aero-acoustic +simulations [@Hasert:2013]. It is written in Fortran, with language constructs from Fortran 2003. @@ -75,13 +81,16 @@ OpenLB [@olbPaper2021] and waLBerla [@BAUER2021478]. Musubi implements the lattice Boltzmann method in the form of kernels that can be run on individual refinement levels of an octree mesh. +It is developed within the APES-Suite [@Klimach:2014] of simulation tools based on the central +Treelm library [@Klimach:2012vi] that provides the handling of this octree mesh on distributed +parallel systems. The interpolation and transformation between the involved levels for the local refinement are separated from the kernel, allowing for an implementation of the respective methods without encumberment by the interpolation between the different resolutions. This method was described in detail in [@hasert:2013jc]. -There are various collision schemes implemented (BGK, MRT, HRR, Cumulants) which can -be used on a range for stencil configurations (discrete velocity directions). +There are various collision schemes implemented (BGK, MRT, HRR, Cumulants [@Spinelli:2023]), +which can be used on a range for stencil configurations (discrete velocity directions). It is also possible to consider the transport of particles and passive scalars in the flow. @@ -90,11 +99,9 @@ the flow. This software has been written by many people over the years. The individual authors can be found in each file with the respective copyright statement. -We want to thank all these contributors, especially the main original developer -of Musubi Manuel Hasert. Not appearing in the list of authors is Sabine Roller, who enabled the development of this software in the first place and we are very grateful for this possibility. -We thank our fellow contributors to this code basis Jiaxing Qi [@Qi:2017], +We especially thank our fellow contributors to this code basis Jiaxing Qi [@Qi:2017], Jens Zudrop [@Zudrop:2015], Simon Zimny [@Zimny:2015], Peter Vitt, Jana Gericke, Tristan Vlogman [@Vlogman:2025], Mengyu Wang and many students. The development of Musubi was partially funded by the German Federal Ministry of Education and Research From aead5159c5b74b744725e3e38525dc41f8d0386d Mon Sep 17 00:00:00 2001 From: haraldkl Date: Fri, 28 Nov 2025 11:17:39 +0100 Subject: [PATCH 15/32] Fix missing comma in paper.bib --- paper.bib | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paper.bib b/paper.bib index 358d1a588..75735e52e 100644 --- a/paper.bib +++ b/paper.bib @@ -153,7 +153,7 @@ @articleInfo{Spinelli:2024 journal = {Discrete and Continuous Dynamical Systems - S}, volume = {17}, number = {11}, - pages = {3224-3251} + pages = {3224-3251}, year = {2024}, issn = {1937-1632}, doi = {10.3934/dcdss.2023212}, From 0f9abab59fc4f73f8e25b418f9a83ed6c1a4d1b6 Mon Sep 17 00:00:00 2001 From: haraldkl Date: Fri, 28 Nov 2025 11:21:30 +0100 Subject: [PATCH 16/32] Fix typo in paper.md --- paper.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paper.md b/paper.md index ae24ff9cf..239f7d923 100644 --- a/paper.md +++ b/paper.md @@ -43,7 +43,7 @@ uses efficient data structures allowing adaptive parallel simulations. Musubi is designed to deal with huge meshes (billions of lattices) and complex geometries on large computing systems efficiently. It can be used for a wide range of application areas from electrodialysis [@Masilamani:2020] -over biomedical problems [@Jain:2016] and aero-dynamic setups [@Spinellis:2024] to aero-acoustic +over biomedical problems [@Jain:2016] and aero-dynamic setups [@Spinelli:2024] to aero-acoustic simulations [@Hasert:2013]. It is written in Fortran, with language constructs from Fortran 2003. From 8a3fa65ae2363cb9202fb2256fb473f0a9a46bfa Mon Sep 17 00:00:00 2001 From: haraldkl Date: Fri, 28 Nov 2025 11:30:10 +0100 Subject: [PATCH 17/32] Add institution in paper.md --- paper.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/paper.md b/paper.md index 239f7d923..e0c14f305 100644 --- a/paper.md +++ b/paper.md @@ -89,7 +89,7 @@ refinement are separated from the kernel, allowing for an implementation of the respective methods without encumberment by the interpolation between the different resolutions. This method was described in detail in [@hasert:2013jc]. -There are various collision schemes implemented (BGK, MRT, HRR, Cumulants [@Spinelli:2023]), +There are various collision schemes implemented (BGK, MRT, HRR, Cumulants) [@Spinelli:2023], which can be used on a range for stencil configurations (discrete velocity directions). It is also possible to consider the transport of particles and passive scalars in the flow. @@ -106,8 +106,10 @@ Jens Zudrop [@Zudrop:2015], Simon Zimny [@Zimny:2015], Peter Vitt, Jana Gericke, Tristan Vlogman [@Vlogman:2025], Mengyu Wang and many students. The development of Musubi was partially funded by the German Federal Ministry of Education and Research (Bundesministerium für Bildung und Forschung, BMBF) in the framework of the HPC software initiative in -the project HISEEM and by the European Commission in the Seventh Framework Programme in the area of -Virtual Physiological Human (THROMBUS project, ICT-2009.5.3, project reference 269966). +the project HISEEM, by the European Commission in the Seventh Framework Programme in the area of +Virtual Physiological Human (THROMBUS project, ICT-2009.5.3, project reference 269966), by the +German Research School of Simulation Sciences, the University of Siegen, the University of Twente +and the German Aerospace Center, DLR. We are grateful for the computing time provided by LRZ in Munich and by HLRS in Stuttgart who also contributed performance evaluations within the POP project. From 5b1611a4acb5405537b5f49f4c6a00d241d3ccaf Mon Sep 17 00:00:00 2001 From: haraldkl Date: Fri, 28 Nov 2025 11:44:59 +0100 Subject: [PATCH 18/32] fix dois in paper.bib --- paper.bib | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/paper.bib b/paper.bib index 75735e52e..11885e4b6 100644 --- a/paper.bib +++ b/paper.bib @@ -112,7 +112,7 @@ @article{Palabos2020 @article{olbPaper2021, author = {Krause, M.J. and Kummerl\"ander, A. and Avis, S.J. and Kusumaatmaja, H. and Dapelo, D. and Klemens, F. and Gaedtke, M. and Hafen, N. and Mink, A. and Trunk, R. and Marquardt, J.E. and Maier, M.L. and Haussmann, M. and Simonis, S.}, title = {{OpenLB--Open source lattice Boltzmann code}}, - doi = {https://doi.org/10.1016/j.camwa.2020.04.033}, + doi = {10.1016/j.camwa.2020.04.033}, issn = {0898-1221}, pages = {258--288}, url = {http://www.sciencedirect.com/science/article/pii/S0898122120301875}, @@ -129,7 +129,7 @@ @article{BAUER2021478 pages = {478-501}, year = {2021}, issn = {0898-1221}, - doi = {https://doi.org/10.1016/j.camwa.2020.01.007}, + doi = {10.1016/j.camwa.2020.01.007}, url = {https://www.sciencedirect.com/science/article/pii/S0898122120300146}, author = {Martin Bauer and Sebastian Eibl and Christian Godenschwager and Nils Kohl and Michael Kuron and Christoph Rettinger and Florian Schornbaum and Christoph Schwarzmeier and Dominik Thönnes and Harald Köstler and Ulrich Rüde}, keywords = {High-performance computing, Multiphysics, Lattice Boltzmann, Rigid particle dynamics, Adaptive mesh refinement, Code generation} @@ -142,7 +142,7 @@ @article{Spinelli:2023 pages = {105833}, year = {2023}, issn = {0045-7930}, - doi = {https://doi.org/10.1016/j.compfluid.2023.105833}, + doi = {10.1016/j.compfluid.2023.105833}, url = {https://www.sciencedirect.com/science/article/pii/S0045793023000580}, author = {Gregorio Gerardo Spinelli and Tobias Horstmann and Kannan Masilamani and Malav Mukesh Soni and Harald Klimach and Arthur Stück and Sabine Roller}, keywords = {Lattice Boltzmann method, Hybrid recursive regularized BGK, Cumulant, Turbulence modeling, Musubi} From ec05c83589d690bd4c90104d6aca72a861251663 Mon Sep 17 00:00:00 2001 From: haraldkl Date: Fri, 28 Nov 2025 11:53:02 +0100 Subject: [PATCH 19/32] Add performance reference to paper.bib --- paper.bib | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/paper.bib b/paper.bib index 11885e4b6..5409bd37b 100644 --- a/paper.bib +++ b/paper.bib @@ -69,6 +69,17 @@ @phdthesis{Jain:2016 Year = {2016} } +@inproceedings{Qi:2016, + author = {Qi, Jiaxing and Jain, Kartik and Klimach, Harald and Roller, Sabine}, + booktitle = {Parallel Computing: On the Road to Exascale}, + editor = {Gerhard R. Joubert and Hugh Leather and Mark Parsons and Frans Peters and Mark Sawyer}, + pages = {807--816}, + series = {Advances in Parallel Computing}, + title = {Performance Evaluation of the LBM Solver Musubi on Various HPC Architectures}, + volume = {27}, + year = {2016} +} + @phdthesis{Qi:2017, Annote = {Supervisor: Roller, Sabine}, Author = {Qi, Jiaxing}, From e61bcdfeb96f5b4599c21fb597ecad2d0c138553 Mon Sep 17 00:00:00 2001 From: haraldkl Date: Fri, 28 Nov 2025 12:00:02 +0100 Subject: [PATCH 20/32] Add Programming in Lua to paper.bib --- paper.bib | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/paper.bib b/paper.bib index 5409bd37b..67e750af9 100644 --- a/paper.bib +++ b/paper.bib @@ -1,3 +1,16 @@ +@Book{Ierusalimschy:2016, + author = {Ierusalimschy, Roberto}, + publisher = {Lua.org}, + title = {Programming in Lua}, + year = {2016}, + address = {Rio de Janeiro}, + edition = {Fourth edition}, + isbn = {9788590379867}, + pagetotal = {369}, + ppn_gvk = {870069926}, +} + + @article{Klimach:2012vi, author = {Klimach, Harald and Hasert, Manuel and Zudrop, Jens and Roller, Sabine}, title = {{Distributed Octree Mesh Infrastructure for Flow Simulations}}, From a12a1aa8efb68cc456fb985621674c1b05f3e4cf Mon Sep 17 00:00:00 2001 From: haraldkl Date: Fri, 28 Nov 2025 12:03:19 +0100 Subject: [PATCH 21/32] Point out computing systems and Lua in paper.md --- paper.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/paper.md b/paper.md index e0c14f305..adb850cf1 100644 --- a/paper.md +++ b/paper.md @@ -93,6 +93,12 @@ There are various collision schemes implemented (BGK, MRT, HRR, Cumulants) [@Spi which can be used on a range for stencil configurations (discrete velocity directions). It is also possible to consider the transport of particles and passive scalars in the flow. +Musubi has a minimal set of dependencies and has been deployed on a wide range of +supercomputing infrastructure ranging from IBM's BlueGene systems to NEC's SX vector +systems [@Qi:2016]. +The user interface is realized via the Lua [@Ierusalimschy:2016] scripting language, +which is used to configure the simulation setups and allows for great flexibility in +the setup definition. # Acknowlegements From b2fb8a0959e0b513b8f18d9eb326c39879686825 Mon Sep 17 00:00:00 2001 From: haraldkl Date: Fri, 28 Nov 2025 12:15:33 +0100 Subject: [PATCH 22/32] Fix typo in title in paper.md --- paper.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paper.md b/paper.md index adb850cf1..07de60f6f 100644 --- a/paper.md +++ b/paper.md @@ -1,5 +1,5 @@ --- -title: 'Musubi: Octree based Lattice-Boltzmann solver for multi-pyhsics' +title: 'Musubi: Octree based Lattice-Boltzmann solver for multi-physics' tags: - Fortran - fluid dynamics From 37829fbe88cc5be7953532ca482511896e62ff11 Mon Sep 17 00:00:00 2001 From: haraldkl Date: Fri, 28 Nov 2025 12:43:42 +0100 Subject: [PATCH 23/32] Add co-author in paper.md --- paper.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/paper.md b/paper.md index 07de60f6f..04331b9c2 100644 --- a/paper.md +++ b/paper.md @@ -11,6 +11,9 @@ authors: - name: Kannan Masilamani orcid: 0000-0002-3640-2154 affiliation: 1 # (Multiple affiliations must be quoted) + - name: Gregorio Gerardo Spinelli + orcid: 0000-0001-6578-2494 + affiliation: 4 - name: Kartik Jain orcid: 0000-0002-6540-9304 affiliation: 2 @@ -27,9 +30,12 @@ affiliations: - name: University of Twente, Netherlands index: 2 ror: 006hf6230 - - name: Festo SE & Co. KG + - name: Festo SE & Co. KG, Germany index: 3 ror: 03ga8q162 + - name: MBDA (Germany), Germany + index: 4 + ror: 05nz79105 date: 28 November 2025 bibliography: paper.bib --- From ee8beefdb78b9254a989dfa17e59272774bd248a Mon Sep 17 00:00:00 2001 From: Harald Klimach Date: Mon, 1 Dec 2025 16:45:13 +0100 Subject: [PATCH 24/32] Add reference to VirtualFluids --- paper.bib | 15 ++++++++++++++- paper.md | 7 +++---- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/paper.bib b/paper.bib index 67e750af9..c9986370a 100644 --- a/paper.bib +++ b/paper.bib @@ -42,7 +42,7 @@ @phdthesis{Hasert:2013 @inproceedings{Klimach:2014, address = {Munich, Germany}, - doi = {10.3233/978-1-61499-381-0-703}, + doi = {10.3233/978-1-61499-381-0-703}, author = {Klimach, Harald and Jain, Kartik and Roller, Sabine}, booktitle = {Parallel Computing: Accelerating Computational Science and Engineering (CSE)}, date-added = {2014-03-05 14:36:31 +0100}, @@ -185,3 +185,16 @@ @articleInfo{Spinelli:2024 author = {Gregorio Gerardo Spinelli and Jana Gericke and Kannan Masilamani and Harald Günther Klimach}, keywords = {Lattice Boltzmann method, Hybrid recursive regularized BGK, Cumulant, turbulence modeling, Musubi, wall functions} } + +@article{Geier:2025, + title = {VirtualFluids – open source parallel LBM solver}, + journal = {Computer Physics Communications}, + volume = {317}, + pages = {109810}, + year = {2025}, + issn = {0010-4655}, + doi = {https://doi.org/10.1016/j.cpc.2025.109810}, + url = {https://www.sciencedirect.com/science/article/pii/S0010465525003121}, + author = {Martin Geier and Konstantin Kutscher and Martin Schönherr and Anna Wellmann and Sören Peters and Hussein Alihussein and Jan Linxweiler and Manfred Krafczyk}, + keywords = {Open source, Lattice Boltzmann method, Cumulant LBM, Grid refinement, Research software engineering, FAIR, HPC, CPU, GPU} +} diff --git a/paper.md b/paper.md index 04331b9c2..11c791649 100644 --- a/paper.md +++ b/paper.md @@ -80,8 +80,8 @@ complicated wall boundaries, as a simple line intersection with the wall geometr can be used to accurately model the surface. Due to these properties the method has gained popularity in the field of computational fluid dynamics over the last decades. -Other Open Source solvers that utilize this method are for example Palabos [@Palabos2020], -OpenLB [@olbPaper2021] and waLBerla [@BAUER2021478]. +Other Open Source solvers that utilize this method are, for example, Palabos [@Palabos2020], +OpenLB [@olbPaper2021], waLBerla [@BAUER2021478] and VirtualFluids [@Geier:2025]. # The Musubi implementation @@ -103,8 +103,7 @@ Musubi has a minimal set of dependencies and has been deployed on a wide range o supercomputing infrastructure ranging from IBM's BlueGene systems to NEC's SX vector systems [@Qi:2016]. The user interface is realized via the Lua [@Ierusalimschy:2016] scripting language, -which is used to configure the simulation setups and allows for great flexibility in -the setup definition. +which is used to configure the simulation setups and allows for great flexibility. # Acknowlegements From 3eeac9b2a9f6b71131126946f854cf61d0eef2f5 Mon Sep 17 00:00:00 2001 From: Harald Klimach Date: Mon, 1 Dec 2025 16:59:34 +0100 Subject: [PATCH 25/32] Add reference to PROTEUS --- paper.bib | 13 +++++++++++++ paper.md | 2 ++ 2 files changed, 15 insertions(+) diff --git a/paper.bib b/paper.bib index c9986370a..1bbb4e300 100644 --- a/paper.bib +++ b/paper.bib @@ -186,6 +186,19 @@ @articleInfo{Spinelli:2024 keywords = {Lattice Boltzmann method, Hybrid recursive regularized BGK, Cumulant, turbulence modeling, Musubi, wall functions} } +@ARTICLE{Blanken:2025, + author={Blanken, Nathan and Heiles, Baptiste and Kuliesh, Alina and Versluis, Michel and Jain, Kartik and Maresca, David and Lajoinie, Guillaume}, + journal={IEEE Transactions on Ultrasonics, Ferroelectrics, and Frequency Control}, + title={PROTEUS: A Physically Realistic Contrast-Enhanced Ultrasound Simulator—Part I: Numerical Methods}, + year={2025}, + volume={72}, + number={7}, + pages={848-865}, + keywords={Ultrasonic imaging;Acoustics;Imaging;Frequency control;Transducers;Propagation;Trajectory;Blood flow measurement;multiphysics simulation;nonlinear wave propagation;open-source software;ultrasound contrast agents (UCAs);ultrasound imaging}, + doi={10.1109/TUFFC.2024.3427850} +} + + @article{Geier:2025, title = {VirtualFluids – open source parallel LBM solver}, journal = {Computer Physics Communications}, diff --git a/paper.md b/paper.md index 11c791649..6494704f5 100644 --- a/paper.md +++ b/paper.md @@ -64,6 +64,8 @@ Musubi implements the lattice Boltzmann method (LBM) with a Message Passing Inte (MPI) parallelization with a fully distributed handling of the mesh data, avoiding bottlenecks on individual processors and enabling the scaling of the simulation to hundreds of thousands of MPI processes. +Musubi is utilized to solve flow problems in the ultrasound simulator PROTEUS developed at +the University of Twente [@Blanken:2025]. # The lattice Boltzmann method From d3a23c3ee8703531f4370614eda521819e621596 Mon Sep 17 00:00:00 2001 From: haraldkl Date: Thu, 4 Dec 2025 10:14:56 +0100 Subject: [PATCH 26/32] Update Acknowledgements in paper.md --- paper.md | 1 + 1 file changed, 1 insertion(+) diff --git a/paper.md b/paper.md index 6494704f5..e8316b319 100644 --- a/paper.md +++ b/paper.md @@ -117,6 +117,7 @@ of this software in the first place and we are very grateful for this possibilit We especially thank our fellow contributors to this code basis Jiaxing Qi [@Qi:2017], Jens Zudrop [@Zudrop:2015], Simon Zimny [@Zimny:2015], Peter Vitt, Jana Gericke, Tristan Vlogman [@Vlogman:2025], Mengyu Wang and many students. +Many thanks to Christian Siebert, who advised us on parallel algorithms and data structures. The development of Musubi was partially funded by the German Federal Ministry of Education and Research (Bundesministerium für Bildung und Forschung, BMBF) in the framework of the HPC software initiative in the project HISEEM, by the European Commission in the Seventh Framework Programme in the area of From b82a5af60742320aae1a3c1198170322ef5963cf Mon Sep 17 00:00:00 2001 From: haraldkl Date: Thu, 4 Dec 2025 10:29:43 +0100 Subject: [PATCH 27/32] Add seeder ref to paper.bib --- paper.bib | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/paper.bib b/paper.bib index 1bbb4e300..32592f582 100644 --- a/paper.bib +++ b/paper.bib @@ -19,6 +19,28 @@ @article{Klimach:2012vi pages = {1--15} } +@InProceedings{Harlacher:2012, + author="Harlacher, Daniel F. + and Hasert, Manuel + and Klimach, Harald + and Zimny, Simon + and Roller, Sabine", + doi = {10.1007/978-3-642-22244-3_6}, + editor="Resch, Michael + and Wang, Xin + and Bez, Wolfgang + and Focht, Erich + and Kobayashi, Hiroaki + and Roller, Sabine", + title="Tree Based Voxelization of STL Data", + booktitle="High Performance Computing on Vector Systems 2011", + year="2012", + publisher="Springer Berlin Heidelberg", + address="Berlin, Heidelberg", + pages="81--92", + isbn="978-3-642-22244-3" +} + @article{hasert:2013jc, author = {Hasert, Manuel and Masilamani, Kannan and Zimny, Simon and Klimach, Harald and Qi, Jiaxing and Bernsdorf, J{\"o}rg and Roller, Sabine}, doi = {10.1016/j.jocs.2013.11.001}, From 00f30dd265609f2578fc7476ce6d4db16a985acc Mon Sep 17 00:00:00 2001 From: haraldkl Date: Thu, 4 Dec 2025 10:39:21 +0100 Subject: [PATCH 28/32] Some corrections and revision to paper.md --- paper.md | 38 ++++++++++++++++++++------------------ 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/paper.md b/paper.md index e8316b319..fdb6f8eba 100644 --- a/paper.md +++ b/paper.md @@ -44,7 +44,7 @@ bibliography: paper.bib Musubi is a multi-level, parallel lattice Boltzmann solver and part of the APES suite. It is working on an octree mesh that is linearized by a (Morton) space-filling curve and -uses efficient data structures allowing adaptive parallel simulations. +uses efficient data structures allowing for adaptive, distributed parallel simulations. Musubi is designed to deal with huge meshes (billions of lattices) and complex geometries on large computing systems efficiently. @@ -52,7 +52,7 @@ It can be used for a wide range of application areas from electrodialysis [@Masi over biomedical problems [@Jain:2016] and aero-dynamic setups [@Spinelli:2024] to aero-acoustic simulations [@Hasert:2013]. -It is written in Fortran, with language constructs from Fortran 2003. +It is written in Fortran, requiring a compiler that provides at least the Fortran 2003 standard. # Statement of need @@ -61,7 +61,7 @@ Due to the nonlinearity and the large amount of degrees of freedom to consider i problems, these simulations require significant computational resources, which typically are only available in distributed parallel systems. Musubi implements the lattice Boltzmann method (LBM) with a Message Passing Interface -(MPI) parallelization with a fully distributed handling of the mesh data, avoiding +(MPI) parallelization and a fully distributed handling of the data, avoiding bottlenecks on individual processors and enabling the scaling of the simulation to hundreds of thousands of MPI processes. Musubi is utilized to solve flow problems in the ultrasound simulator PROTEUS developed at @@ -69,13 +69,13 @@ the University of Twente [@Blanken:2025]. # The lattice Boltzmann method -The lattice Boltzmann method utilizes ideas of cellular automata and represents at its -core a basic two step algorithm. -The state of the fluid is represented by particle density functions (PDF) in a discrete +The lattice Boltzmann method employs ideas of cellular automata and can be represented +at its core as a basic two step algorithm. +The state of the fluid is represented by particle density functions (PDF) of a discrete velocity field. These PDFs reside on the lattices and are exchanged along the discrete velocity directions. -The two steps of the algorithm are now the streaming of the PDF information along -the velocity directions, followed by the so-called collision, computing the new +The two steps of the algorithm are the streaming of the PDF information along +velocity directions, followed by the so-called collision, computing a new PDF on each lattice. This modeling with discrete velocities also allows for a straight forward handling of complicated wall boundaries, as a simple line intersection with the wall geometry @@ -92,32 +92,34 @@ run on individual refinement levels of an octree mesh. It is developed within the APES-Suite [@Klimach:2014] of simulation tools based on the central Treelm library [@Klimach:2012vi] that provides the handling of this octree mesh on distributed parallel systems. +The dedicated meshing tool Seeder [@Harlacher:2012] provides this octree mesh in a format that +enables the distributed parallel reading of mesh partitions by all processes. The interpolation and transformation between the involved levels for the local refinement are separated from the kernel, allowing for an implementation of the respective methods without encumberment by the interpolation between the different resolutions. This method was described in detail in [@hasert:2013jc]. There are various collision schemes implemented (BGK, MRT, HRR, Cumulants) [@Spinelli:2023], -which can be used on a range for stencil configurations (discrete velocity directions). -It is also possible to consider the transport of particles and passive scalars in -the flow. -Musubi has a minimal set of dependencies and has been deployed on a wide range of -supercomputing infrastructure ranging from IBM's BlueGene systems to NEC's SX vector -systems [@Qi:2016]. +which can be used on a range of stencil configurations (discrete velocity directions). +It is also possible to consider the transport of particles [@Vlogman:2025] and passive +scalars in the flow. +Musubi has a minimal set of dependencies and has been deployed on a varity of +supercomputing systems ranging from IBM's BlueGene to NEC's SX vector systems [@Qi:2016]. The user interface is realized via the Lua [@Ierusalimschy:2016] scripting language, -which is used to configure the simulation setups and allows for great flexibility. +which is used to configure the simulation setups and allows for great flexibility in +problem definitions. # Acknowlegements This software has been written by many people over the years. The individual authors can be found in each file with the respective copyright statement. -Not appearing in the list of authors is Sabine Roller, who enabled the development +Not appearing in those lists of authors is Sabine Roller, who enabled the development of this software in the first place and we are very grateful for this possibility. We especially thank our fellow contributors to this code basis Jiaxing Qi [@Qi:2017], Jens Zudrop [@Zudrop:2015], Simon Zimny [@Zimny:2015], Peter Vitt, Jana Gericke, -Tristan Vlogman [@Vlogman:2025], Mengyu Wang and many students. -Many thanks to Christian Siebert, who advised us on parallel algorithms and data structures. +Tristan Vlogman, Mengyu Wang and many students. +Further, we thank Christian Siebert, who advised us on parallel algorithms and data structures. The development of Musubi was partially funded by the German Federal Ministry of Education and Research (Bundesministerium für Bildung und Forschung, BMBF) in the framework of the HPC software initiative in the project HISEEM, by the European Commission in the Seventh Framework Programme in the area of From 28db224e174095cc2d78f5f74631ebecc21ceb70 Mon Sep 17 00:00:00 2001 From: Harald Klimach Date: Thu, 12 Feb 2026 15:58:11 +0100 Subject: [PATCH 29/32] JOSS: Add newly requested scope sections --- paper.bib | 88 +++++++++++++++++++++++++++++++++++++++++++++++++++++-- paper.md | 72 +++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 156 insertions(+), 4 deletions(-) diff --git a/paper.bib b/paper.bib index 32592f582..e93676a87 100644 --- a/paper.bib +++ b/paper.bib @@ -62,6 +62,20 @@ @phdthesis{Hasert:2013 Year = {2013} } +@article{Zimny:2013, + author = {Zimny, Simon and Chopard, Bastien and Malaspinas, Orestis and Lorenz, Eric and Jain, Kartik and Roller, Sabine and Bernsdorf, J{\"o}rg}, + doi = {10.1016/j.procs.2013.05.266}, + issn = {1877-0509}, + journal = {Procedia Computer Science}, + keywords = {Multiscale Simulation, Lattice Boltzmann Method, Scale Separation Map, Thrombus, Intracranial Aneurysms}, + note = {2013 International Conference on Computational Science}, + pages = {1006--1015}, + title = {A Multiscale Approach for the Coupled Simulation of Blood Flow and Thrombus Formation in Intracranial Aneurysms}, + volume = {18}, + year = {2013}, +} + + @inproceedings{Klimach:2014, address = {Munich, Germany}, doi = {10.3233/978-1-61499-381-0-703}, @@ -96,11 +110,50 @@ @phdthesis{Zudrop:2015 Year = {2015} } +@inproceedings{Qi:2015, + author = {Qi, Jiaxing and Hasert, Manuel and Klimach, Harald and Roller, Sabine}, + booktitle = {Sustained Simulation Performance 2015}, + doi = {10.1007/978-3-319-20340-9_16}, + editor = {Resch, Michael and Bez, Wolfgang and Focht, Erich and Kobayashi, Hiroaki and Qi, Jiaxing and Roller, Sabine}, + pages = {195--204}, + publisher = {Springer International Publishing}, + title = {Aeroacoustic Simulation of Flow Through Porous Media Based on Lattice Boltzmann Method}, + year = {2015} +} + + +@article{Johannink:2015, +title = {Predictive pressure drop models for membrane channels with non-woven and woven spacers}, +journal = {Desalination}, +volume = {376}, +pages = {41-54}, +year = {2015}, +issn = {0011-9164}, +doi = {10.1016/j.desal.2015.07.024}, +url = {https://www.sciencedirect.com/science/article/pii/S0011916415300321}, +author = {Matthias Johannink and Kannan Masilamani and Adel Mhamdi and Sabine Roller and Wolfgang Marquardt}, +keywords = {Membrane processes, Spacer, Pressure drop, Modeling}, +} + +@article{JainCaF:2016, + annote = {Journal Papers}, + author = {Jain, Kartik and Roller, Sabine and Mardal, Kent-Andr{\'e}}, + doi = {doi:10.1016/j.compfluid.2015.12.011}, + journal = {Computers \& Fluids}, + month = {mar}, + pages = {36--46}, + publisher = {Pergamon}, + title = {Transitional flow in intracranial aneurysms--a space and time refinement study below the Kolmogorov scales using Lattice Boltzmann Method}, + volume = {127}, + year = {2016} +} + @phdthesis{Jain:2016, Annote = {Supervisor: Roller, Sabine}, Author = {Jain, Kartik}, School = {Universit{\"a}t Siegen}, Title = {Transition to Turbulence in Physiological Flows: Direct Numerical Simulation of Hemodynamics in Intracranial Aneurysms and Cerebrospinal Fluid Hydrodynamics in the Spinal Canal}, + isbn = {978-3-936533-83-5}, Year = {2016} } @@ -115,6 +168,22 @@ @inproceedings{Qi:2016 year = {2016} } +@article{Masilamani:2017, + annote = {Journal Papers}, + author = {Zudrop, Jens and Masilamani, Kannan and Roller, Sabine, and Asinari, Pietro}, + doi = {10.1016/j.compfluid.2017.04.021}, + issn = {0045-7930}, + journal = {Computers and Fluids}, + keywords = {Lattice Boltzmann; Multicomponent flow; Maxwell--Stefan diffusion; High performance computing; Massively parallel simulations; Octree}, + pages = {20-33}, + publisher = {Elsevier}, + title = {A robust lattice Boltzmann method for parallel simulations of multicomponent flows in complex geometries}, + url = {http://www.sciencedirect.com/science/article/pii/S0045793017301482}, + volume = {153}, + year = {2017}, +} + + @phdthesis{Qi:2017, Annote = {Supervisor: Roller, Sabine}, Author = {Qi, Jiaxing}, @@ -131,6 +200,21 @@ @phdthesis{Masilamani:2020 Year = {2020} } +@article{Hebbink:2022, + title = {Computational analysis of human upper airway aerodynamics}, + volume = {61}, + ISSN = {1741-0444}, + url = {http://dx.doi.org/10.1007/s11517-022-02716-8}, + DOI = {10.1007/s11517-022-02716-8}, + number = {2}, + journal = {Medical & Biological Engineering & Computing}, + publisher = {Springer Science and Business Media LLC}, + author = {Hebbink, Rutger H.J. and Wessels, Bas J. and Hagmeijer, Rob and Jain, Kartik}, + year = {2022}, + month = dec, + pages = {541–553} +} + @phdthesis{Vlogman:2025, title = "Coupled Lattice Boltzmann and discrete element method simulations with applications to liver radioembolization", author = "Tristan Vlogman", @@ -151,7 +235,7 @@ @article{Palabos2020 journal = "Computers & Mathematics with Applications", year = "2020", issn = "0898-1221", - doi = "https://doi.org/10.1016/j.camwa.2020.03.022", + doi = "10.1016/j.camwa.2020.03.022", author = "Jonas Latt and Orestis Malaspinas and Dimitrios Kontaxakis and Andrea Parmigiani and Daniel Lagrava and Federico Brogi and Mohamed Ben Belgacem and Yann Thorimbert and Sébastien Leclaire and Sha Li and Francesco Marson and Jonathan Lemus and Christos Kotsalos and Raphaël Conradin and Christophe Coreixas and Rémy Petkantchin and Franck Raynaud and Joël Beny and Bastien Chopard", } @@ -228,7 +312,7 @@ @article{Geier:2025 pages = {109810}, year = {2025}, issn = {0010-4655}, - doi = {https://doi.org/10.1016/j.cpc.2025.109810}, + doi = {10.1016/j.cpc.2025.109810}, url = {https://www.sciencedirect.com/science/article/pii/S0010465525003121}, author = {Martin Geier and Konstantin Kutscher and Martin Schönherr and Anna Wellmann and Sören Peters and Hussein Alihussein and Jan Linxweiler and Manfred Krafczyk}, keywords = {Open source, Lattice Boltzmann method, Cumulant LBM, Grid refinement, Research software engineering, FAIR, HPC, CPU, GPU} diff --git a/paper.md b/paper.md index fdb6f8eba..5ec343f9d 100644 --- a/paper.md +++ b/paper.md @@ -67,6 +67,71 @@ hundreds of thousands of MPI processes. Musubi is utilized to solve flow problems in the ultrasound simulator PROTEUS developed at the University of Twente [@Blanken:2025]. +# State of the field + +There is a wide range of computational fluid dynamic methods, of which the lattice Boltzmann +method represents an attractive option, due to its low number of operations and straight +forward explicit implementation. + +Other Open Source solvers that utilize this method are, for example, Palabos [@Palabos2020], +OpenLB [@olbPaper2021], waLBerla [@BAUER2021478] and VirtualFluids [@Geier:2025]. +They all aim at scale-resolved simulations of fluids. +Musubi was developed (since 2011) in parallel or predated some of those projects +due to a ground-up orientation to make use of octree meshes on massively parallel computing systems +with a dedicated mesh format that allows for a distributed parallel reading from the file +system. +This approach allows for an automated multi-level mesh generation and avoids parallelisation +bottlenecks between the mesh generation step and the simulation. + +A specific domain that is addressed in Musubi and not covered by other Open Source LBM solvers, +is the simulation of the Maxwell-Stefan equation for multiple species [@Masilamani:2017] as needed +in diffusion processes that appear for example in electrodialysis applications. + +# Software design + +Musubi is developed in the Apes-Suite framework, which revolves around a central octree +mesh representation. +This central part is implemented in a separate library, shared by the different +tools in the framework. +Though there still is a tight development dependency that is expressed by the use +of git submodules. +The application is designed towards deployment on a wide range of high-performance +computing systems, including more exotic architectures like IBM's BlueGene or NEC's +SX vector systems. +Hence, dependencies are kept to a minimum and utilized Fortran language features are those, +that are commonly supported. +As user interface in this environment, the scripting language Lua is chosen, which allows +for a flexible configuration of simulation setups but does not introduce complicated +dependencies, as Lua is implemented in standard ANSI C and is compiled along with the +project. + +Musubi separates the kernels, implementing the lattice-Boltzmann method on a single +mesh refinement level from the supporting infrastructure that takes care of interpolation +and communication. +The goal here is to enable rapid implementation of new methods unperturbed by the +infrastructure details. + +# Research impact statement + +Musubi has been successfully deployed in the simulation of scale-resolved fluid +problems for a variety of problems. +Originally, a main funding source for the software was the German HISEEM project that +aimed at the investigation of effective electrodialysis processes for seawater +desalination [@Johannink:2015]. +Another main focus in the development is put on biomedical application [@JainCaF:2016], +which was originally supported by the European THROMBUS project [@Zimny:2013]. +In the biodmedical domain Musubi now is also used as the fluid simulation tool in +the ultrasound simulator PROTEUS developed at the University of Twente [@Blanken:2025]. + +While the two aforementioned application domains primarily are concerned with +the simulation of liquids, there are also applications to gaseous fluids, +for example in the simulation of human upper airway aerodynamics [@Hebbink:2022]. +Musubi's application extends beyond these domains and has been used in +general aerodynamic simulations [@Spinelli:2024], aswell as in the domain of +aero-acoustics [@Qi:2015], where the resolution of multiple spatial scales plays an +important role. + + # The lattice Boltzmann method The lattice Boltzmann method employs ideas of cellular automata and can be represented @@ -82,8 +147,6 @@ complicated wall boundaries, as a simple line intersection with the wall geometr can be used to accurately model the surface. Due to these properties the method has gained popularity in the field of computational fluid dynamics over the last decades. -Other Open Source solvers that utilize this method are, for example, Palabos [@Palabos2020], -OpenLB [@olbPaper2021], waLBerla [@BAUER2021478] and VirtualFluids [@Geier:2025]. # The Musubi implementation @@ -109,6 +172,11 @@ The user interface is realized via the Lua [@Ierusalimschy:2016] scripting langu which is used to configure the simulation setups and allows for great flexibility in problem definitions. +# AI usage disclosure + +No generative AI tools were used in the development of this software, the writing of this +manuscript, or the preparation of supporting materials. + # Acknowlegements This software has been written by many people over the years. From 61869e58a67a36c834e8f10f6677b4854cb8ea45 Mon Sep 17 00:00:00 2001 From: haraldkl Date: Fri, 13 Feb 2026 15:55:12 +0100 Subject: [PATCH 30/32] Add DOI fields to bibliography entries --- paper.bib | 3 +++ 1 file changed, 3 insertions(+) diff --git a/paper.bib b/paper.bib index e93676a87..9abe20b33 100644 --- a/paper.bib +++ b/paper.bib @@ -162,6 +162,7 @@ @inproceedings{Qi:2016 booktitle = {Parallel Computing: On the Road to Exascale}, editor = {Gerhard R. Joubert and Hugh Leather and Mark Parsons and Frans Peters and Mark Sawyer}, pages = {807--816}, + doi = {10.3233/978-1-61499-621-7-807}, series = {Advances in Parallel Computing}, title = {Performance Evaluation of the LBM Solver Musubi on Various HPC Architectures}, volume = {27}, @@ -189,6 +190,7 @@ @phdthesis{Qi:2017 Author = {Qi, Jiaxing}, School = {RWTH Aachen University}, Title = {Efficient Lattice Boltzmann Simulations on Large Scale High Performance Computing Systems}, + doi = {10.18154/RWTH-2017-04121}, Year = {2017} } @@ -197,6 +199,7 @@ @phdthesis{Masilamani:2020 Author = {Masilamani, Kannan}, School = {RWTH Aachen University}, Title = {Framework for Coupled Simulation of Electrodialysis Processes}, + doi = {10.18154/RWTH-2021-02437}, Year = {2020} } From 3da5d3dddc193cbbe430583291b8e0b95141bdec Mon Sep 17 00:00:00 2001 From: haraldkl Date: Fri, 13 Feb 2026 16:42:09 +0100 Subject: [PATCH 31/32] joss: Fix typo in Acknowledgements section header --- paper.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paper.md b/paper.md index 5ec343f9d..a65c2ec0b 100644 --- a/paper.md +++ b/paper.md @@ -177,7 +177,7 @@ problem definitions. No generative AI tools were used in the development of this software, the writing of this manuscript, or the preparation of supporting materials. -# Acknowlegements +# Acknowledgements This software has been written by many people over the years. The individual authors can be found in each file with the respective copyright From ba55123f0f0c8d623b9eb9337a84c44b1e1ac044 Mon Sep 17 00:00:00 2001 From: haraldkl Date: Fri, 13 Feb 2026 17:06:52 +0100 Subject: [PATCH 32/32] Merge Musubi section into Design * Moved the lattice Boltzmann section further up. * Removed redundant sections and improved clarity on software design and research impact. --- paper.md | 100 ++++++++++++++++++++++++++----------------------------- 1 file changed, 47 insertions(+), 53 deletions(-) diff --git a/paper.md b/paper.md index a65c2ec0b..ecbf08039 100644 --- a/paper.md +++ b/paper.md @@ -54,6 +54,7 @@ simulations [@Hasert:2013]. It is written in Fortran, requiring a compiler that provides at least the Fortran 2003 standard. + # Statement of need Highly resolved fluid simulations are an integral part in many scientifc application areas. @@ -67,6 +68,24 @@ hundreds of thousands of MPI processes. Musubi is utilized to solve flow problems in the ultrasound simulator PROTEUS developed at the University of Twente [@Blanken:2025]. + +# The lattice Boltzmann method + +The lattice Boltzmann method employs ideas of cellular automata and can be represented +at its core as a basic two step algorithm. +The state of the fluid is represented by particle density functions (PDF) of a discrete +velocity field. +These PDFs reside on the lattices and are exchanged along the discrete velocity directions. +The two steps of the algorithm are the streaming of the PDF information along +velocity directions, followed by the so-called collision, computing a new +PDF on each lattice. +This modeling with discrete velocities also allows for a straight forward handling of +complicated wall boundaries, as a simple line intersection with the wall geometry +can be used to accurately model the surface. +Due to these properties the method has gained popularity in the field of computational +fluid dynamics over the last decades. + + # State of the field There is a wide range of computational fluid dynamic methods, of which the lattice Boltzmann @@ -87,29 +106,41 @@ A specific domain that is addressed in Musubi and not covered by other Open Sour is the simulation of the Maxwell-Stefan equation for multiple species [@Masilamani:2017] as needed in diffusion processes that appear for example in electrodialysis applications. + # Software design -Musubi is developed in the Apes-Suite framework, which revolves around a central octree -mesh representation. +Musubi implements the lattice Boltzmann method in the form of kernels that can be +run on individual refinement levels of an octree mesh. +It is developed within the APES-Suite [@Klimach:2014] of simulation tools revolving around a central +Treelm library [@Klimach:2012vi] that provides the handling of this octree mesh on distributed +parallel systems. This central part is implemented in a separate library, shared by the different tools in the framework. Though there still is a tight development dependency that is expressed by the use of git submodules. + +The dedicated meshing tool Seeder [@Harlacher:2012] provides this octree mesh in a format that +enables the distributed parallel reading of mesh partitions by all processes. +The interpolation and transformation between the involved levels for the local +refinement are separated from the kernel, allowing for an implementation of the +respective methods without encumberment by the interpolation between the different +resolutions. +This method was described in detail in [@hasert:2013jc] and enables the rapid +implementation of new numerical kernels. +There are various collision schemes implemented (BGK, MRT, HRR, Cumulants) [@Spinelli:2023], +which can be used on a range of stencil configurations (discrete velocity directions). +It is also possible to consider the transport of particles [@Vlogman:2025] and passive +scalars in the flow. + The application is designed towards deployment on a wide range of high-performance -computing systems, including more exotic architectures like IBM's BlueGene or NEC's -SX vector systems. -Hence, dependencies are kept to a minimum and utilized Fortran language features are those, -that are commonly supported. -As user interface in this environment, the scripting language Lua is chosen, which allows -for a flexible configuration of simulation setups but does not introduce complicated +computing systems. +To facilitate this, Musubi is designed with a minimal set of dependencies allowing for deployment +on a varity of supercomputing systems ranging from IBM's BlueGene to NEC's SX vector systems [@Qi:2016]. +As user interface in this environment, the scripting language Lua [@Ierusalimschy:2016] is chosen, +which allows for a flexible configuration of simulation setups but does not introduce complicated dependencies, as Lua is implemented in standard ANSI C and is compiled along with the project. -Musubi separates the kernels, implementing the lattice-Boltzmann method on a single -mesh refinement level from the supporting infrastructure that takes care of interpolation -and communication. -The goal here is to enable rapid implementation of new methods unperturbed by the -infrastructure details. # Research impact statement @@ -130,53 +161,16 @@ Musubi's application extends beyond these domains and has been used in general aerodynamic simulations [@Spinelli:2024], aswell as in the domain of aero-acoustics [@Qi:2015], where the resolution of multiple spatial scales plays an important role. +The distributed handling of octree meshes is well suited in these settings as +it allows for the resolution of the scales that need to be resolved. -# The lattice Boltzmann method - -The lattice Boltzmann method employs ideas of cellular automata and can be represented -at its core as a basic two step algorithm. -The state of the fluid is represented by particle density functions (PDF) of a discrete -velocity field. -These PDFs reside on the lattices and are exchanged along the discrete velocity directions. -The two steps of the algorithm are the streaming of the PDF information along -velocity directions, followed by the so-called collision, computing a new -PDF on each lattice. -This modeling with discrete velocities also allows for a straight forward handling of -complicated wall boundaries, as a simple line intersection with the wall geometry -can be used to accurately model the surface. -Due to these properties the method has gained popularity in the field of computational -fluid dynamics over the last decades. - -# The Musubi implementation - -Musubi implements the lattice Boltzmann method in the form of kernels that can be -run on individual refinement levels of an octree mesh. -It is developed within the APES-Suite [@Klimach:2014] of simulation tools based on the central -Treelm library [@Klimach:2012vi] that provides the handling of this octree mesh on distributed -parallel systems. -The dedicated meshing tool Seeder [@Harlacher:2012] provides this octree mesh in a format that -enables the distributed parallel reading of mesh partitions by all processes. -The interpolation and transformation between the involved levels for the local -refinement are separated from the kernel, allowing for an implementation of the -respective methods without encumberment by the interpolation between the different -resolutions. -This method was described in detail in [@hasert:2013jc]. -There are various collision schemes implemented (BGK, MRT, HRR, Cumulants) [@Spinelli:2023], -which can be used on a range of stencil configurations (discrete velocity directions). -It is also possible to consider the transport of particles [@Vlogman:2025] and passive -scalars in the flow. -Musubi has a minimal set of dependencies and has been deployed on a varity of -supercomputing systems ranging from IBM's BlueGene to NEC's SX vector systems [@Qi:2016]. -The user interface is realized via the Lua [@Ierusalimschy:2016] scripting language, -which is used to configure the simulation setups and allows for great flexibility in -problem definitions. - # AI usage disclosure No generative AI tools were used in the development of this software, the writing of this manuscript, or the preparation of supporting materials. + # Acknowledgements This software has been written by many people over the years.