From 7ccf187d7df348dfbef274ff9c1fa6d5b557d46e Mon Sep 17 00:00:00 2001 From: Thomas Flynn Date: Thu, 2 Jul 2026 18:47:36 +0100 Subject: [PATCH 1/5] some initial section headings for MPI comms issues --- distributed-memory/collective.tex | 8 ++++++++ distributed-memory/point-to-point.tex | 8 ++++++++ distributed-memory/small-messages.tex | 3 +++ 3 files changed, 19 insertions(+) create mode 100644 distributed-memory/collective.tex create mode 100644 distributed-memory/point-to-point.tex create mode 100644 distributed-memory/small-messages.tex diff --git a/distributed-memory/collective.tex b/distributed-memory/collective.tex new file mode 100644 index 0000000..e8d7b88 --- /dev/null +++ b/distributed-memory/collective.tex @@ -0,0 +1,8 @@ +\section{Collective communication inefficiencies} +\label{section:distributed-memory:collective} + +\subsection{Late broadcaster} + +\subsection{Early reduce} + +\subsection{Wait at NxN} diff --git a/distributed-memory/point-to-point.tex b/distributed-memory/point-to-point.tex new file mode 100644 index 0000000..591eed9 --- /dev/null +++ b/distributed-memory/point-to-point.tex @@ -0,0 +1,8 @@ +\section{Point-to-point communication inefficiencies} +\label{section:distributed-memory:point-to-point} + +\subsection{Late senders} + +\subsection{Late receivers} + +\subsection{Wrong order} diff --git a/distributed-memory/small-messages.tex b/distributed-memory/small-messages.tex new file mode 100644 index 0000000..4acfb95 --- /dev/null +++ b/distributed-memory/small-messages.tex @@ -0,0 +1,3 @@ +\section{Small messages} +\label{section:distributed-memory:small-messages} + From 972b4347510022da121d947a313c958e4ec5ade6 Mon Sep 17 00:00:00 2001 From: Thomas Flynn Date: Thu, 2 Jul 2026 18:57:51 +0100 Subject: [PATCH 2/5] adding in initial MPI files to main --- main.tex | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/main.tex b/main.tex index 926ebc6..c1fe7a4 100644 --- a/main.tex +++ b/main.tex @@ -88,6 +88,11 @@ \part{Deep Dive} \input{node-level/hierarchical-decomposition} \input{node-level/fine-granular-task-serialisation} +\input{distributed-memory/distributed-memory} +\input{distributed-memory/small-messages} +\input{distributed-memory/point-to-point} +\input{distributed-memory/collective} + \part{Resources} \label{part:resources} From c7fb8917f5a3b819db025037320235cb983194dd Mon Sep 17 00:00:00 2001 From: Thomas Flynn Date: Thu, 2 Jul 2026 18:58:06 +0100 Subject: [PATCH 3/5] placeholder for intro to MPI section --- distributed-memory/distributed-memory.tex | 62 +++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 distributed-memory/distributed-memory.tex diff --git a/distributed-memory/distributed-memory.tex b/distributed-memory/distributed-memory.tex new file mode 100644 index 0000000..76872e7 --- /dev/null +++ b/distributed-memory/distributed-memory.tex @@ -0,0 +1,62 @@ +\chapter{Distributed-memory parallelism (inter-node) analysis} +\label{chapter:node-level} + + + +%Reaching this section indicates that we are struggling to scale across multiple nodes. +%This is evidenced by the fact that our intra-node scaling is not good (cmp.~Section~\ref{section:high-level:inter-node}), +%i.e.~we measure what we get on one core and then extrapolate what our given number of cores should yield. +%But they don't: +%The speedup as introduced from \eqref{equation:hpc-terms:speedup} is poor as we increase the core count. +%We leave a lot of potential compute capability unused. + + + +%Our deep dive starts from an analysis if there is enough concurrency in principle to exploit all cores. +%If there are not enough concurrent operations, further analysis is not required. +%The code can be returned immediately to the developers. +%Otherwise, we take it from there (Figure~\ref{figure:core:decision-graph}). + + +%\subsection*{Preparation} + +%Inter-node analysis is all about how efficiently we use multiple threads. +%Therefore, it focuses predominantly on relative quantities: +%How well does the code perform if we add another thread? +%However, we cannot study 128+ measurements in one go---even though modern performance analysis tools have quite powerful capabilities to compare different setups. +%We need to focus on few setups that yield interesting setup. + + + +%\begin{assessmentcrime} +% People study a whole-node run right from the start without a scalability +% analysis beforehand. +%\end{assessmentcrime} + + + +%\noindent +%It makes sense to run a strong scaling test and plot the efficiency. +%Once completed, we can flag those areas where the gradient suddenly changes or +%where the efficiency drops below 80\% or 60\% respectively. +%These are the setups (turning points) on which we should focus most of the time. +%Do not try to study any other setup, as you'll most likely be swamped with data +%that is not relevant or difficult to interpret! + + + +\paragraph*{How it works} + +\begin{itemize}[noitemsep] + %\item[$\boxplus$] Run a weak scaling test and plot the efficiency. Narrow down the number of setups of interest, i.e.~identify regions where the efficiency apruptly changes or falls under a threshold. + \item[$\boxplus$] TBD +\end{itemize} + + +\paragraph*{How it does not work} + +\begin{itemize}[noitemsep] + %\item[$\boxminus$] Start with a full-node run right from the start and hope that you spot interesting trends in the data. + \item[$\boxminus$] TBD +\end{itemize} + From ede6148cf4988d87f63d3246ef42e0bd32a2dc9e Mon Sep 17 00:00:00 2001 From: Thomas Flynn Date: Tue, 7 Jul 2026 16:12:54 +0100 Subject: [PATCH 4/5] adding some structure to imbalances --- book.bib | 11 +++++++++++ distributed-memory/communication-imbalance.tex | 6 ++++++ distributed-memory/workload-imbalance.tex | 6 ++++++ 3 files changed, 23 insertions(+) create mode 100644 distributed-memory/communication-imbalance.tex create mode 100644 distributed-memory/workload-imbalance.tex diff --git a/book.bib b/book.bib index 5487427..69e357e 100644 --- a/book.bib +++ b/book.bib @@ -88,6 +88,15 @@ @inproceedings{geimer2007 year={2007} } +% MPI-focused performance - Characterising load and communication imbalance +@book{bohme2014, + title={Characterizing load and communication imbalance in parallel applications}, + author={B{\"o}hme, David}, + volume={23}, + year={2014}, + publisher={Forschungszentrum J{\"u}lich} +} + % Peak performance reference - Emily 2026-03-16 @incollection{sterling_chapter_1_2018, address = {Boston}, @@ -123,3 +132,5 @@ @incollection{sterling_chapter_15_2018 pages = {451--481}, file = {PDF:C\:\\Users\\MDS_ghvn82_\$\\Zotero\\storage\\ZLITG4MC\\Sterling et al. - 2018 - Chapter 15 - Accelerator Architecture.pdf:application/pdf}, } + + diff --git a/distributed-memory/communication-imbalance.tex b/distributed-memory/communication-imbalance.tex new file mode 100644 index 0000000..d518cbd --- /dev/null +++ b/distributed-memory/communication-imbalance.tex @@ -0,0 +1,6 @@ +\section{Communication imbalance} +\label{section:distributed-memory:communication-imbalance} + +\subsection{Varying number of MPI calls} + +\subsection{Varying size of messages} diff --git a/distributed-memory/workload-imbalance.tex b/distributed-memory/workload-imbalance.tex new file mode 100644 index 0000000..6871eff --- /dev/null +++ b/distributed-memory/workload-imbalance.tex @@ -0,0 +1,6 @@ +\section{Workload imbalance} +\label{section:distributed-memory:workload-imbalance} + +\subsection{Static workload imbalance} + +\subsection{Dynamic workload imbalance} From 67466f5ca19d27422c79027efcab2cb5d586c107 Mon Sep 17 00:00:00 2001 From: Thomas Flynn Date: Tue, 7 Jul 2026 16:13:35 +0100 Subject: [PATCH 5/5] adding scan to communication imbalance --- distributed-memory/collective.tex | 2 ++ 1 file changed, 2 insertions(+) diff --git a/distributed-memory/collective.tex b/distributed-memory/collective.tex index e8d7b88..43539c1 100644 --- a/distributed-memory/collective.tex +++ b/distributed-memory/collective.tex @@ -6,3 +6,5 @@ \subsection{Late broadcaster} \subsection{Early reduce} \subsection{Wait at NxN} + +\subsection{Early scan}