diff --git a/.github/workflows/draft-pdf.yml b/.github/workflows/draft-pdf.yml new file mode 100644 index 000000000..1d7dc44e4 --- /dev/null +++ b/.github/workflows/draft-pdf.yml @@ -0,0 +1,24 @@ +name: Draft PDF +on: [push] + +jobs: + paper: + runs-on: ubuntu-latest + name: Paper Draft + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Build draft PDF + uses: openjournals/openjournals-draft-action@master + with: + journal: joss + # This should be the path to the paper within your repo. + paper-path: paper.md + - name: Upload + uses: actions/upload-artifact@v4 + with: + name: paper + # This is the output path where Pandoc will write the compiled + # PDF. Note, this should be the same directory as the input + # paper.md + path: paper.pdf diff --git a/paper.bib b/paper.bib new file mode 100644 index 000000000..9abe20b33 --- /dev/null +++ b/paper.bib @@ -0,0 +1,322 @@ +@Book{Ierusalimschy:2016, + author = {Ierusalimschy, Roberto}, + publisher = {Lua.org}, + title = {Programming in Lua}, + year = {2016}, + address = {Rio de Janeiro}, + edition = {Fourth edition}, + isbn = {9788590379867}, + pagetotal = {369}, + ppn_gvk = {870069926}, +} + + +@article{Klimach:2012vi, + author = {Klimach, Harald and Hasert, Manuel and Zudrop, Jens and Roller, Sabine}, + title = {{Distributed Octree Mesh Infrastructure for Flow Simulations}}, + journal = {European Congress on Computational Methods in Applied Sciences and Engineering}, + year = {2012}, + pages = {1--15} +} + +@InProceedings{Harlacher:2012, + author="Harlacher, Daniel F. + and Hasert, Manuel + and Klimach, Harald + and Zimny, Simon + and Roller, Sabine", + doi = {10.1007/978-3-642-22244-3_6}, + editor="Resch, Michael + and Wang, Xin + and Bez, Wolfgang + and Focht, Erich + and Kobayashi, Hiroaki + and Roller, Sabine", + title="Tree Based Voxelization of STL Data", + booktitle="High Performance Computing on Vector Systems 2011", + year="2012", + publisher="Springer Berlin Heidelberg", + address="Berlin, Heidelberg", + pages="81--92", + isbn="978-3-642-22244-3" +} + +@article{hasert:2013jc, + author = {Hasert, Manuel and Masilamani, Kannan and Zimny, Simon and Klimach, Harald and Qi, Jiaxing and Bernsdorf, J{\"o}rg and Roller, Sabine}, + doi = {10.1016/j.jocs.2013.11.001}, + issn = {1877-7503}, + journal = {Journal of Computational Science}, + month = {sep}, + number = {5}, + pages = {784--794}, + title = {Complex fluid simulations with the parallel tree-based {Lattice} {Boltzmann} solver {Musubi}}, + volume = {5}, + year = {2014} +} + +@phdthesis{Hasert:2013, + Annote = {Supervisor: Roller, Sabine}, + Author = {Hasert, Manuel}, + School = {RWTH Aachen University}, + Title = {Multi-Scale Lattice Boltzmann Simulations on Distributed Octrees}, + Year = {2013} +} + +@article{Zimny:2013, + author = {Zimny, Simon and Chopard, Bastien and Malaspinas, Orestis and Lorenz, Eric and Jain, Kartik and Roller, Sabine and Bernsdorf, J{\"o}rg}, + doi = {10.1016/j.procs.2013.05.266}, + issn = {1877-0509}, + journal = {Procedia Computer Science}, + keywords = {Multiscale Simulation, Lattice Boltzmann Method, Scale Separation Map, Thrombus, Intracranial Aneurysms}, + note = {2013 International Conference on Computational Science}, + pages = {1006--1015}, + title = {A Multiscale Approach for the Coupled Simulation of Blood Flow and Thrombus Formation in Intracranial Aneurysms}, + volume = {18}, + year = {2013}, +} + + +@inproceedings{Klimach:2014, + address = {Munich, Germany}, + doi = {10.3233/978-1-61499-381-0-703}, + author = {Klimach, Harald and Jain, Kartik and Roller, Sabine}, + booktitle = {Parallel Computing: Accelerating Computational Science and Engineering (CSE)}, + date-added = {2014-03-05 14:36:31 +0100}, + date-modified = {2016-04-21 07:20:21 +0000}, + editor = {Bader, Michael and Bode, Arndt and Bungartz, Hans-Joachim and Gerndt, Michael and Joubert, Gerhard R. and Peters, Frans}, + month = {sep}, + pages = {703--711}, + publisher = {IOS Press}, + series = {Advances in Parallel Computing}, + title = {End-to-end parallel simulations with APES}, + volume = {25}, + year = {2014} +} + + +@phdthesis{Zimny:2015, + Annote = {Supervisor: Roller, Sabine}, + Author = {Zimny, Simon}, + School = {RWTH Aachen University}, + Title = {Hemodynamic Flow Simulation in Patient Specific Cerebral Aneurysms}, + Year = {2015} +} + +@phdthesis{Zudrop:2015, + Annote = {Supervisor: Roller, Sabine}, + Author = {Zudrop, Jens}, + School = {RWTH Aachen University}, + Title = {Efficient Numerical Methods for Fluid- and Electrodynamics on Massively Parallel Systems}, + Year = {2015} +} + +@inproceedings{Qi:2015, + author = {Qi, Jiaxing and Hasert, Manuel and Klimach, Harald and Roller, Sabine}, + booktitle = {Sustained Simulation Performance 2015}, + doi = {10.1007/978-3-319-20340-9_16}, + editor = {Resch, Michael and Bez, Wolfgang and Focht, Erich and Kobayashi, Hiroaki and Qi, Jiaxing and Roller, Sabine}, + pages = {195--204}, + publisher = {Springer International Publishing}, + title = {Aeroacoustic Simulation of Flow Through Porous Media Based on Lattice Boltzmann Method}, + year = {2015} +} + + +@article{Johannink:2015, +title = {Predictive pressure drop models for membrane channels with non-woven and woven spacers}, +journal = {Desalination}, +volume = {376}, +pages = {41-54}, +year = {2015}, +issn = {0011-9164}, +doi = {10.1016/j.desal.2015.07.024}, +url = {https://www.sciencedirect.com/science/article/pii/S0011916415300321}, +author = {Matthias Johannink and Kannan Masilamani and Adel Mhamdi and Sabine Roller and Wolfgang Marquardt}, +keywords = {Membrane processes, Spacer, Pressure drop, Modeling}, +} + +@article{JainCaF:2016, + annote = {Journal Papers}, + author = {Jain, Kartik and Roller, Sabine and Mardal, Kent-Andr{\'e}}, + doi = {doi:10.1016/j.compfluid.2015.12.011}, + journal = {Computers \& Fluids}, + month = {mar}, + pages = {36--46}, + publisher = {Pergamon}, + title = {Transitional flow in intracranial aneurysms--a space and time refinement study below the Kolmogorov scales using Lattice Boltzmann Method}, + volume = {127}, + year = {2016} +} + +@phdthesis{Jain:2016, + Annote = {Supervisor: Roller, Sabine}, + Author = {Jain, Kartik}, + School = {Universit{\"a}t Siegen}, + Title = {Transition to Turbulence in Physiological Flows: Direct Numerical Simulation of Hemodynamics in Intracranial Aneurysms and Cerebrospinal Fluid Hydrodynamics in the Spinal Canal}, + isbn = {978-3-936533-83-5}, + Year = {2016} +} + +@inproceedings{Qi:2016, + author = {Qi, Jiaxing and Jain, Kartik and Klimach, Harald and Roller, Sabine}, + booktitle = {Parallel Computing: On the Road to Exascale}, + editor = {Gerhard R. Joubert and Hugh Leather and Mark Parsons and Frans Peters and Mark Sawyer}, + pages = {807--816}, + doi = {10.3233/978-1-61499-621-7-807}, + series = {Advances in Parallel Computing}, + title = {Performance Evaluation of the LBM Solver Musubi on Various HPC Architectures}, + volume = {27}, + year = {2016} +} + +@article{Masilamani:2017, + annote = {Journal Papers}, + author = {Zudrop, Jens and Masilamani, Kannan and Roller, Sabine, and Asinari, Pietro}, + doi = {10.1016/j.compfluid.2017.04.021}, + issn = {0045-7930}, + journal = {Computers and Fluids}, + keywords = {Lattice Boltzmann; Multicomponent flow; Maxwell--Stefan diffusion; High performance computing; Massively parallel simulations; Octree}, + pages = {20-33}, + publisher = {Elsevier}, + title = {A robust lattice Boltzmann method for parallel simulations of multicomponent flows in complex geometries}, + url = {http://www.sciencedirect.com/science/article/pii/S0045793017301482}, + volume = {153}, + year = {2017}, +} + + +@phdthesis{Qi:2017, + Annote = {Supervisor: Roller, Sabine}, + Author = {Qi, Jiaxing}, + School = {RWTH Aachen University}, + Title = {Efficient Lattice Boltzmann Simulations on Large Scale High Performance Computing Systems}, + doi = {10.18154/RWTH-2017-04121}, + Year = {2017} +} + +@phdthesis{Masilamani:2020, + Annote = {Supervisor: Roller, Sabine}, + Author = {Masilamani, Kannan}, + School = {RWTH Aachen University}, + Title = {Framework for Coupled Simulation of Electrodialysis Processes}, + doi = {10.18154/RWTH-2021-02437}, + Year = {2020} +} + +@article{Hebbink:2022, + title = {Computational analysis of human upper airway aerodynamics}, + volume = {61}, + ISSN = {1741-0444}, + url = {http://dx.doi.org/10.1007/s11517-022-02716-8}, + DOI = {10.1007/s11517-022-02716-8}, + number = {2}, + journal = {Medical & Biological Engineering & Computing}, + publisher = {Springer Science and Business Media LLC}, + author = {Hebbink, Rutger H.J. and Wessels, Bas J. and Hagmeijer, Rob and Jain, Kartik}, + year = {2022}, + month = dec, + pages = {541–553} +} + +@phdthesis{Vlogman:2025, + title = "Coupled Lattice Boltzmann and discrete element method simulations with applications to liver radioembolization", + author = "Tristan Vlogman", + year = "2025", + month = jul, + day = "11", + doi = "10.3990/1.9789036567046", + isbn = "978-90-365-6703-9", + publisher = "University of Twente", + address = "Netherlands", + type = "PhD Thesis - Research UT, graduation UT", + school = "University of Twente" +} + + +@article{Palabos2020, + title = "Palabos: Parallel Lattice Boltzmann Solver", + journal = "Computers & Mathematics with Applications", + year = "2020", + issn = "0898-1221", + doi = "10.1016/j.camwa.2020.03.022", + author = "Jonas Latt and Orestis Malaspinas and Dimitrios Kontaxakis and Andrea Parmigiani and Daniel Lagrava and Federico Brogi and Mohamed Ben Belgacem and Yann Thorimbert and Sébastien Leclaire and Sha Li and Francesco Marson and Jonathan Lemus and Christos Kotsalos and Raphaël Conradin and Christophe Coreixas and Rémy Petkantchin and Franck Raynaud and Joël Beny and Bastien Chopard", +} + +@article{olbPaper2021, + author = {Krause, M.J. and Kummerl\"ander, A. and Avis, S.J. and Kusumaatmaja, H. and Dapelo, D. and Klemens, F. and Gaedtke, M. and Hafen, N. and Mink, A. and Trunk, R. and Marquardt, J.E. and Maier, M.L. and Haussmann, M. and Simonis, S.}, + title = {{OpenLB--Open source lattice Boltzmann code}}, + doi = {10.1016/j.camwa.2020.04.033}, + issn = {0898-1221}, + pages = {258--288}, + url = {http://www.sciencedirect.com/science/article/pii/S0898122120301875}, + volume = {81}, + journal = {Computers \& Mathematics with Applications}, + keywords = {Numerical simulation, Lattice Boltzmann methods, Partial differential equations, Computational fluid dynamics, Transport problems, OpenLB}, + year = {2021} +} + +@article{BAUER2021478, + title = {waLBerla: A block-structured high-performance framework for multiphysics simulations}, + journal = {Computers & Mathematics with Applications}, + volume = {81}, + pages = {478-501}, + year = {2021}, + issn = {0898-1221}, + doi = {10.1016/j.camwa.2020.01.007}, + url = {https://www.sciencedirect.com/science/article/pii/S0898122120300146}, + author = {Martin Bauer and Sebastian Eibl and Christian Godenschwager and Nils Kohl and Michael Kuron and Christoph Rettinger and Florian Schornbaum and Christoph Schwarzmeier and Dominik Thönnes and Harald Köstler and Ulrich Rüde}, + keywords = {High-performance computing, Multiphysics, Lattice Boltzmann, Rigid particle dynamics, Adaptive mesh refinement, Code generation} +} + +@article{Spinelli:2023, + title = {HPC performance study of different collision models using the Lattice Boltzmann solver Musubi}, + journal = {Computers & Fluids}, + volume = {255}, + pages = {105833}, + year = {2023}, + issn = {0045-7930}, + doi = {10.1016/j.compfluid.2023.105833}, + url = {https://www.sciencedirect.com/science/article/pii/S0045793023000580}, + author = {Gregorio Gerardo Spinelli and Tobias Horstmann and Kannan Masilamani and Malav Mukesh Soni and Harald Klimach and Arthur Stück and Sabine Roller}, + keywords = {Lattice Boltzmann method, Hybrid recursive regularized BGK, Cumulant, Turbulence modeling, Musubi} +} + +@articleInfo{Spinelli:2024, + title = {Key ingredients for wall-modeled LES with the Lattice Boltzmann method: Systematic comparison of collision schemes, SGS models, and wall functions on simulation accuracy and efficiency for turbulent channel flow}, + journal = {Discrete and Continuous Dynamical Systems - S}, + volume = {17}, + number = {11}, + pages = {3224-3251}, + year = {2024}, + issn = {1937-1632}, + doi = {10.3934/dcdss.2023212}, + url = {https://www.aimsciences.org/article/id/6578251d4aaea52710702d59}, + author = {Gregorio Gerardo Spinelli and Jana Gericke and Kannan Masilamani and Harald Günther Klimach}, + keywords = {Lattice Boltzmann method, Hybrid recursive regularized BGK, Cumulant, turbulence modeling, Musubi, wall functions} +} + +@ARTICLE{Blanken:2025, + author={Blanken, Nathan and Heiles, Baptiste and Kuliesh, Alina and Versluis, Michel and Jain, Kartik and Maresca, David and Lajoinie, Guillaume}, + journal={IEEE Transactions on Ultrasonics, Ferroelectrics, and Frequency Control}, + title={PROTEUS: A Physically Realistic Contrast-Enhanced Ultrasound Simulator—Part I: Numerical Methods}, + year={2025}, + volume={72}, + number={7}, + pages={848-865}, + keywords={Ultrasonic imaging;Acoustics;Imaging;Frequency control;Transducers;Propagation;Trajectory;Blood flow measurement;multiphysics simulation;nonlinear wave propagation;open-source software;ultrasound contrast agents (UCAs);ultrasound imaging}, + doi={10.1109/TUFFC.2024.3427850} +} + + +@article{Geier:2025, + title = {VirtualFluids – open source parallel LBM solver}, + journal = {Computer Physics Communications}, + volume = {317}, + pages = {109810}, + year = {2025}, + issn = {0010-4655}, + doi = {10.1016/j.cpc.2025.109810}, + url = {https://www.sciencedirect.com/science/article/pii/S0010465525003121}, + author = {Martin Geier and Konstantin Kutscher and Martin Schönherr and Anna Wellmann and Sören Peters and Hussein Alihussein and Jan Linxweiler and Manfred Krafczyk}, + keywords = {Open source, Lattice Boltzmann method, Cumulant LBM, Grid refinement, Research software engineering, FAIR, HPC, CPU, GPU} +} diff --git a/paper.md b/paper.md new file mode 100644 index 000000000..ecbf08039 --- /dev/null +++ b/paper.md @@ -0,0 +1,194 @@ +--- +title: 'Musubi: Octree based Lattice-Boltzmann solver for multi-physics' +tags: + - Fortran + - fluid dynamics + - Lattice-Boltzmann method + - Maxwell-Stefan equations + - particle transport + - distributed parallelism +authors: + - name: Kannan Masilamani + orcid: 0000-0002-3640-2154 + affiliation: 1 # (Multiple affiliations must be quoted) + - name: Gregorio Gerardo Spinelli + orcid: 0000-0001-6578-2494 + affiliation: 4 + - name: Kartik Jain + orcid: 0000-0002-6540-9304 + affiliation: 2 + - name: Manuel Hasert + affiliation: 3 + - name: Harald Klimach + orcid: 0000-0002-6054-5681 + corresponding: true # (This is how to denote the corresponding author) + affiliation: 1 +affiliations: + - name: German Aerospace Center, DLR, Germany + index: 1 + ror: 04bwf3e34 + - name: University of Twente, Netherlands + index: 2 + ror: 006hf6230 + - name: Festo SE & Co. KG, Germany + index: 3 + ror: 03ga8q162 + - name: MBDA (Germany), Germany + index: 4 + ror: 05nz79105 +date: 28 November 2025 +bibliography: paper.bib +--- + +# Summary + +Musubi is a multi-level, parallel lattice Boltzmann solver and part of the APES suite. +It is working on an octree mesh that is linearized by a (Morton) space-filling curve and +uses efficient data structures allowing for adaptive, distributed parallel simulations. + +Musubi is designed to deal with huge meshes (billions of lattices) and complex geometries +on large computing systems efficiently. +It can be used for a wide range of application areas from electrodialysis [@Masilamani:2020] +over biomedical problems [@Jain:2016] and aero-dynamic setups [@Spinelli:2024] to aero-acoustic +simulations [@Hasert:2013]. + +It is written in Fortran, requiring a compiler that provides at least the Fortran 2003 standard. + + +# Statement of need + +Highly resolved fluid simulations are an integral part in many scientifc application areas. +Due to the nonlinearity and the large amount of degrees of freedom to consider in these +problems, these simulations require significant computational resources, which typically +are only available in distributed parallel systems. +Musubi implements the lattice Boltzmann method (LBM) with a Message Passing Interface +(MPI) parallelization and a fully distributed handling of the data, avoiding +bottlenecks on individual processors and enabling the scaling of the simulation to +hundreds of thousands of MPI processes. +Musubi is utilized to solve flow problems in the ultrasound simulator PROTEUS developed at +the University of Twente [@Blanken:2025]. + + +# The lattice Boltzmann method + +The lattice Boltzmann method employs ideas of cellular automata and can be represented +at its core as a basic two step algorithm. +The state of the fluid is represented by particle density functions (PDF) of a discrete +velocity field. +These PDFs reside on the lattices and are exchanged along the discrete velocity directions. +The two steps of the algorithm are the streaming of the PDF information along +velocity directions, followed by the so-called collision, computing a new +PDF on each lattice. +This modeling with discrete velocities also allows for a straight forward handling of +complicated wall boundaries, as a simple line intersection with the wall geometry +can be used to accurately model the surface. +Due to these properties the method has gained popularity in the field of computational +fluid dynamics over the last decades. + + +# State of the field + +There is a wide range of computational fluid dynamic methods, of which the lattice Boltzmann +method represents an attractive option, due to its low number of operations and straight +forward explicit implementation. + +Other Open Source solvers that utilize this method are, for example, Palabos [@Palabos2020], +OpenLB [@olbPaper2021], waLBerla [@BAUER2021478] and VirtualFluids [@Geier:2025]. +They all aim at scale-resolved simulations of fluids. +Musubi was developed (since 2011) in parallel or predated some of those projects +due to a ground-up orientation to make use of octree meshes on massively parallel computing systems +with a dedicated mesh format that allows for a distributed parallel reading from the file +system. +This approach allows for an automated multi-level mesh generation and avoids parallelisation +bottlenecks between the mesh generation step and the simulation. + +A specific domain that is addressed in Musubi and not covered by other Open Source LBM solvers, +is the simulation of the Maxwell-Stefan equation for multiple species [@Masilamani:2017] as needed +in diffusion processes that appear for example in electrodialysis applications. + + +# Software design + +Musubi implements the lattice Boltzmann method in the form of kernels that can be +run on individual refinement levels of an octree mesh. +It is developed within the APES-Suite [@Klimach:2014] of simulation tools revolving around a central +Treelm library [@Klimach:2012vi] that provides the handling of this octree mesh on distributed +parallel systems. +This central part is implemented in a separate library, shared by the different +tools in the framework. +Though there still is a tight development dependency that is expressed by the use +of git submodules. + +The dedicated meshing tool Seeder [@Harlacher:2012] provides this octree mesh in a format that +enables the distributed parallel reading of mesh partitions by all processes. +The interpolation and transformation between the involved levels for the local +refinement are separated from the kernel, allowing for an implementation of the +respective methods without encumberment by the interpolation between the different +resolutions. +This method was described in detail in [@hasert:2013jc] and enables the rapid +implementation of new numerical kernels. +There are various collision schemes implemented (BGK, MRT, HRR, Cumulants) [@Spinelli:2023], +which can be used on a range of stencil configurations (discrete velocity directions). +It is also possible to consider the transport of particles [@Vlogman:2025] and passive +scalars in the flow. + +The application is designed towards deployment on a wide range of high-performance +computing systems. +To facilitate this, Musubi is designed with a minimal set of dependencies allowing for deployment +on a varity of supercomputing systems ranging from IBM's BlueGene to NEC's SX vector systems [@Qi:2016]. +As user interface in this environment, the scripting language Lua [@Ierusalimschy:2016] is chosen, +which allows for a flexible configuration of simulation setups but does not introduce complicated +dependencies, as Lua is implemented in standard ANSI C and is compiled along with the +project. + + +# Research impact statement + +Musubi has been successfully deployed in the simulation of scale-resolved fluid +problems for a variety of problems. +Originally, a main funding source for the software was the German HISEEM project that +aimed at the investigation of effective electrodialysis processes for seawater +desalination [@Johannink:2015]. +Another main focus in the development is put on biomedical application [@JainCaF:2016], +which was originally supported by the European THROMBUS project [@Zimny:2013]. +In the biodmedical domain Musubi now is also used as the fluid simulation tool in +the ultrasound simulator PROTEUS developed at the University of Twente [@Blanken:2025]. + +While the two aforementioned application domains primarily are concerned with +the simulation of liquids, there are also applications to gaseous fluids, +for example in the simulation of human upper airway aerodynamics [@Hebbink:2022]. +Musubi's application extends beyond these domains and has been used in +general aerodynamic simulations [@Spinelli:2024], aswell as in the domain of +aero-acoustics [@Qi:2015], where the resolution of multiple spatial scales plays an +important role. +The distributed handling of octree meshes is well suited in these settings as +it allows for the resolution of the scales that need to be resolved. + + +# AI usage disclosure + +No generative AI tools were used in the development of this software, the writing of this +manuscript, or the preparation of supporting materials. + + +# Acknowledgements + +This software has been written by many people over the years. +The individual authors can be found in each file with the respective copyright +statement. +Not appearing in those lists of authors is Sabine Roller, who enabled the development +of this software in the first place and we are very grateful for this possibility. +We especially thank our fellow contributors to this code basis Jiaxing Qi [@Qi:2017], +Jens Zudrop [@Zudrop:2015], Simon Zimny [@Zimny:2015], Peter Vitt, Jana Gericke, +Tristan Vlogman, Mengyu Wang and many students. +Further, we thank Christian Siebert, who advised us on parallel algorithms and data structures. +The development of Musubi was partially funded by the German Federal Ministry of Education and Research +(Bundesministerium für Bildung und Forschung, BMBF) in the framework of the HPC software initiative in +the project HISEEM, by the European Commission in the Seventh Framework Programme in the area of +Virtual Physiological Human (THROMBUS project, ICT-2009.5.3, project reference 269966), by the +German Research School of Simulation Sciences, the University of Siegen, the University of Twente +and the German Aerospace Center, DLR. +We are grateful for the computing time provided by LRZ in Munich and by HLRS in Stuttgart who +also contributed performance evaluations within the POP project. + +# References