Add support for precompiled headers

configure.py

@@ -250,6 +250,7 @@ def __init__(self, source_paths, options, modules):
         self.libobj_dir = os.path.join(self.build_dir, 'obj', 'lib')
         self.cliobj_dir = os.path.join(self.build_dir, 'obj', 'cli')
         self.testobj_dir = os.path.join(self.build_dir, 'obj', 'test')
+        self.pch_dir = os.path.join(self.build_dir, 'obj', 'pch') if options.enable_pch else None
 
         self.doc_output_dir = os.path.join(self.build_dir, 'docs')
         self.handbook_output_dir = os.path.join(self.doc_output_dir, 'handbook')
@@ -327,6 +328,8 @@ def build_dirs(self):
             out += [self.fuzzobj_dir, self.fuzzer_output_dir]
         if self.example_output_dir:
             out += [self.example_obj_dir, self.example_output_dir]
+        if self.pch_dir:
+            out += [self.pch_dir]
         return out
 
     def format_public_include_flags(self, cc):
@@ -564,6 +567,8 @@ def add_enable_disable_pair(group, what, default, msg=optparse.SUPPRESS_HELP):
                            default=True, action='store_false',
                            help=optparse.SUPPRESS_HELP)
 
+    add_enable_disable_pair(build_group, 'pch', True, 'disable precompiled headers')
+
     add_with_without_pair(build_group, 'valgrind', False, 'use valgrind API')
 
     build_group.add_option('--unsafe-fuzzer-mode', action='store_true', default=False,
@@ -1325,6 +1330,9 @@ def __init__(self, infofile):
                 'ninja_header_deps_style': '',
                 'header_deps_flag': '',
                 'header_deps_out': '',
+                'pch_compile': None,
+                'pch_suffix': None,
+                'pch_include': None,
             })
 
         self.add_framework_option = lex.add_framework_option
@@ -1373,6 +1381,9 @@ def __init__(self, infofile):
         self.header_deps_flag = lex.header_deps_flag
         self.header_deps_out = lex.header_deps_out
         self.ct_value_barrier = lex.ct_value_barrier
+        self.pch_compile = lex.pch_compile
+        self.pch_suffix = lex.pch_suffix
+        self.pch_include = lex.pch_include
 
     def cross_check(self, os_info, arch_info, all_isas):
 
@@ -1419,17 +1430,15 @@ def gen_lib_flags(self, options, variables):
         """
 
         def flag_builder():
-            # We always emit -fPIC or equivalent so that position independent executables
-            # can be created that link to the static library
-            yield self.shared_flags
+            # -fPIC or equiv is emitted via cc_compile_flags so that it's used for all objects
+            # (this is mostly for the benefit of being able to share the PCH)
 
             if options.build_shared_lib:
                 yield self.visibility_build_flags
 
             if 'debug' in self.lib_flags and options.with_debug_info:
                 yield process_template_string(self.lib_flags['debug'], variables, self.infofile)
 
-
         return ' '.join(list(flag_builder()))
 
     def gen_visibility_attribute(self, options):
@@ -1458,6 +1467,12 @@ def ct_value_barrier_type(self, options):
 
         return None
 
+    def supports_pch(self):
+        if self.pch_compile is not None:
+            return self.pch_compile != ''
+        else:
+            return False
+
     def mach_abi_link_flags(self, options, debug_mode=None):
 
         """
@@ -1561,12 +1576,17 @@ def cc_compile_flags(self, options):
         if options.cxxflags:
             # CXXFLAGS is assumed to be the entire set of desired compilation flags
             # if not the case the user should have used --extra-cxxflags
+            if self.shared_flags != '':
+                yield self.shared_flags
             yield options.cxxflags
             return
 
         if options.with_debug_info:
             yield self.debug_info_flags
 
+        if self.shared_flags != '':
+            yield self.shared_flags
+
         if not options.no_optimizations:
             if options.optimize_for_size:
                 if self.size_optimization_flags != '':
@@ -2248,6 +2268,10 @@ def test_exe_extra_ldflags():
         'with_doxygen': options.with_doxygen,
         'maintainer_mode': options.maintainer_mode,
 
+        'enable_pch': options.enable_pch and cc.supports_pch(),
+        'pch_suffix': cc.pch_suffix or '',
+        'pch_compile': cc.pch_compile or '',
+
         'out_dir': normalize_source_path(build_dir),
         'build_dir': normalize_source_path(build_paths.build_dir),
         'module_info_dir': build_paths.doc_module_info,
@@ -2421,6 +2445,29 @@ def test_exe_extra_ldflags():
 
     variables['lib_flags'] = cc.gen_lib_flags(options, variables)
 
+    if variables['enable_pch']:
+        variables['pch_dir'] = build_paths.pch_dir
+        variables['pch_src_header'] = os.path.join(source_paths.lib_dir, 'pch/pch.h')
+        variables['pch_header_for_lib'] = os.path.join(build_paths.pch_dir, 'pch_lib.h')
+        variables['pch_include_for_lib'] = '%s %s' % (cc.pch_include, variables['pch_header_for_lib'])
+        variables['pch_path_for_lib'] = variables['pch_header_for_lib'] + '.' + cc.pch_suffix
+
+        variables['pch_header_for_exe'] = os.path.join(build_paths.pch_dir, 'pch_exe.h')
+        variables['pch_include_for_exe'] = '%s %s' % (cc.pch_include, variables['pch_header_for_exe'])
+        variables['pch_path_for_exe'] = variables['pch_header_for_exe'] + '.' + cc.pch_suffix
+
+        variables['pch_target'] = 'pch'
+    else:
+        variables['pch_src_header'] = ''
+        variables['pch_header_for_lib'] = ''
+        variables['pch_include_for_lib'] = ''
+        variables['pch_path_for_lib'] = ''
+        variables['pch_header_for_exe'] = ''
+        variables['pch_include_for_exe'] = ''
+        variables['pch_path_for_exe'] = ''
+
+        variables['pch_target'] = ''
+
     if options.with_pkg_config:
         variables['botan_pkgconfig'] = os.path.join(build_paths.build_dir, 'botan-%d.pc' % (Version.major()))
     if options.with_cmake_config:

doc/dev_ref/contributing.rst

@@ -154,6 +154,9 @@ If you don't already use it for all your C/C++ development, install ``ccache``
 (or on Windows, ``sccache``) right now, and configure a large cache on a fast
 disk. It allows for very quick rebuilds by caching the compiler output.
 
+For ``ccache``, setting ``CCACHE_SLOPPINESS=pch_defines,time_macros`` allows
+using both the compiler cache and precompiled headers, for best compilation speed.
+
 Use ``--enable-sanitizers=`` flag to enable various sanitizer checks.  Supported
 values including "address" and "undefined" for GCC and Clang. GCC also supports
 "iterator" (checked iterators), and Clang supports "memory" (MSan) and

src/build-data/cc/clang.txt

@@ -21,6 +21,10 @@ size_optimization_flags "-Os"
 add_system_include_dir_option "-isystem"
 add_sysroot_option "--sysroot="
 
+pch_suffix "pch"
+pch_compile "-Xclang -fno-pch-timestamp -x c++-header"
+pch_include "-Xclang -fno-pch-timestamp -include"
+
 <sanitizers>
 default   -> address,undefined
 

src/build-data/cc/gcc.txt

@@ -7,13 +7,13 @@ minimum_supported_version 11.0
 lang_flags "-std=c++20 -D_REENTRANT"
 
 # This should only contain flags which are included in GCC 11
-warning_flags "-Wall -Wextra -Wpedantic -Wstrict-aliasing -Wcast-align -Wmissing-declarations -Wpointer-arith -Wcast-qual -Wzero-as-null-pointer-constant -Wnon-virtual-dtor -Wold-style-cast -Wsuggest-override -Wshadow -Wextra-semi"
+warning_flags "-Wall -Wextra -Wpedantic -Wstrict-aliasing -Wcast-align -Wmissing-declarations -Wpointer-arith -Wcast-qual -Wzero-as-null-pointer-constant -Wnon-virtual-dtor -Wold-style-cast -Wsuggest-override -Wshadow -Wextra-semi -Winvalid-pch"
 
 # Boost headers have 0 as nullptr and non-virtual-dtor issues so we can't werror on them
 # -Wmaybe-uninitialized is buggy https://gcc.gnu.org/bugzilla/show_bug.cgi?id=105937
 # -Wstringop-overread is buggy https://gcc.gnu.org/bugzilla/show_bug.cgi?id=111499
 # -Wfree-nonheap-object is buggy https://gcc.gnu.org/bugzilla/show_bug.cgi?id=115016
-werror_flags "-Werror -Wno-error=strict-overflow -Wno-error=zero-as-null-pointer-constant -Wno-error=non-virtual-dtor -Wno-error=maybe-uninitialized -Wno-error=stringop-overread -Wno-error=stringop-overflow -Wno-error=free-nonheap-object -Wno-error=restrict"
+werror_flags "-Werror -Wno-error=strict-overflow -Wno-error=zero-as-null-pointer-constant -Wno-error=non-virtual-dtor -Wno-error=maybe-uninitialized -Wno-error=stringop-overread -Wno-error=stringop-overflow -Wno-error=free-nonheap-object -Wno-error=array-bounds -Wno-error=restrict"
 
 maintainer_warning_flags "-Wstrict-overflow=5"
 
@@ -30,6 +30,10 @@ stack_protector_flags "-fstack-protector"
 add_system_include_dir_option "-isystem"
 add_sysroot_option "--sysroot="
 
+pch_compile "-x c++-header"
+pch_suffix "gch"
+pch_include "-include"
+
 <sanitizers>
 default   -> iterator,address
 

src/build-data/cc/xcode.txt

@@ -21,6 +21,10 @@ size_optimization_flags "-Os"
 add_system_include_dir_option "-isystem"
 add_sysroot_option "--sysroot="
 
+pch_suffix "pch"
+pch_compile "-Xclang -fno-pch-timestamp -x c++-header"
+pch_include "-Xclang -fno-pch-timestamp -include"
+
 <sanitizers>
 default   -> address,undefined
 

src/build-data/makefile.in

@@ -51,6 +51,22 @@ docs: %{doc_stamp_file}
 .PHONY: all cli libs tests check docs clean distclean install fmt tidy
 %{endif}
 
+%{if enable_pch}
+
+%{pch_header_for_lib}: %{pch_src_header}
+	"$(PYTHON_EXE)" -c "import shutil, sys; shutil.copyfile(sys.argv[1], sys.argv[2])" "$<" "$@"
+
+%{pch_header_for_exe}: %{pch_src_header}
+	"$(PYTHON_EXE)" -c "import shutil, sys; shutil.copyfile(sys.argv[1], sys.argv[2])" "$<" "$@"
+
+%{pch_path_for_lib}: %{pch_header_for_lib}
+	$(CXX) $(LIB_FLAGS) $(BUILD_FLAGS) -DBOTAN_IS_BEING_BUILT %{public_include_flags} %{internal_include_flags} %{external_include_flags} %{pch_compile} %{dash_c} %{pch_header_for_lib} %{dash_o}$@
+
+%{pch_path_for_exe}: %{pch_header_for_exe}
+	$(CXX) $(BUILD_FLAGS) -DBOTAN_IS_BEING_BUILT %{public_include_flags} %{internal_include_flags} %{external_include_flags} %{pch_compile} %{dash_c} %{pch_header_for_exe} %{dash_o}$@
+
+%{endif}
+
 %{doc_stamp_file}: %{doc_dir}/*.rst %{doc_dir}/api_ref/*.rst %{doc_dir}/dev_ref/*.rst
 	"$(PYTHON_EXE)" "$(SCRIPTS_DIR)/build_docs.py" --build-dir="%{build_dir}"
 
@@ -150,18 +166,18 @@ ct_selftest: %{out_dir}/botan_ct_selftest
 # Build Commands
 
 %{for lib_build_info}
-%{obj}: %{src}
-	$(CXX) $(LIB_FLAGS) $(BUILD_FLAGS) -DBOTAN_IS_BEING_BUILT %{public_include_flags} %{internal_include_flags} %{external_include_flags} %{dash_c} %{src} %{dash_o}$@
+%{obj}: %{pch_path_for_lib} %{src}
+	$(CXX) %{pch_include_for_lib} $(LIB_FLAGS) $(BUILD_FLAGS) -DBOTAN_IS_BEING_BUILT %{public_include_flags} %{internal_include_flags} %{external_include_flags} %{dash_c} %{src} %{dash_o}$@
 %{endfor}
 
 %{for cli_build_info}
-%{obj}: %{src}
-	$(CXX) $(BUILD_FLAGS) -DBOTAN_IS_BEING_BUILT %{public_include_flags} %{internal_include_flags} %{external_include_flags} %{dash_c} %{src} %{dash_o}$@
+%{obj}: %{pch_path_for_exe} %{src}
+	$(CXX) %{pch_include_for_exe} $(BUILD_FLAGS) -DBOTAN_IS_BEING_BUILT %{public_include_flags} %{internal_include_flags} %{external_include_flags} %{dash_c} %{src} %{dash_o}$@
 %{endfor}
 
 %{for test_build_info}
-%{obj}: %{src}
-	$(CXX) $(BUILD_FLAGS) -DBOTAN_IS_BEING_BUILT %{public_include_flags} %{internal_include_flags} %{external_include_flags} %{dash_c} %{src} %{dash_o}$@
+%{obj}: %{pch_path_for_exe} %{src}
+	$(CXX) %{pch_include_for_exe} $(BUILD_FLAGS) -DBOTAN_IS_BEING_BUILT %{public_include_flags} %{internal_include_flags} %{external_include_flags} %{dash_c} %{src} %{dash_o}$@
 %{endfor}
 
 %{for fuzzer_build_info}

src/build-data/ninja.in

@@ -21,14 +21,34 @@ EXE_LINKS_TO        = %{link_to_botan} ${LIB_LINKS_TO} %{extra_libs}
 SCRIPTS_DIR    = %{scripts_dir}
 INSTALLED_LIB_DIR = %{libdir}
 
+%{if enable_pch}
+rule copy_pch_header
+  command = "${PYTHON_EXE}" -c "import shutil, sys; shutil.copyfile(sys.argv[1], sys.argv[2])" "$in" "$out"
+
+rule compile_pch_lib
+  command = %{cxx} %{lib_flags} ${ABI_FLAGS} ${LANG_FLAGS} ${CXXFLAGS} -DBOTAN_IS_BEING_BUILT ${WARN_FLAGS} %{public_include_flags} %{internal_include_flags} %{external_include_flags} %{pch_compile} %{dash_c} $in %{dash_o}$out
+
+rule compile_pch_exe
+  command = %{cxx} ${ABI_FLAGS} ${LANG_FLAGS} ${CXXFLAGS} -DBOTAN_IS_BEING_BUILT ${WARN_FLAGS} %{public_include_flags} %{internal_include_flags} %{external_include_flags} %{pch_compile} %{dash_c} $in %{dash_o}$out
+
+build %{pch_header_for_lib}: copy_pch_header %{pch_src_header}
+
+build %{pch_header_for_exe}: copy_pch_header %{pch_src_header}
+
+build %{pch_path_for_lib}: compile_pch_lib %{pch_header_for_lib}
+
+build %{pch_path_for_exe}: compile_pch_exe %{pch_header_for_exe}
+
+%{endif}
+
 rule compile_lib
 %{if header_deps_out}
   depfile = $out.d
 %{endif}
 %{if ninja_header_deps_style}
   deps = %{ninja_header_deps_style}
 %{endif}
-  command = %{cxx} %{lib_flags} ${ABI_FLAGS} ${LANG_FLAGS} ${CXXFLAGS} -DBOTAN_IS_BEING_BUILT ${WARN_FLAGS} %{public_include_flags} %{internal_include_flags} %{external_include_flags} %{header_deps_flag} %{header_deps_out|concat: $out.d} %{dash_c} $in %{dash_o}$out
+  command = %{cxx} %{pch_include_for_lib} %{lib_flags} ${ABI_FLAGS} ${LANG_FLAGS} ${CXXFLAGS} -DBOTAN_IS_BEING_BUILT ${WARN_FLAGS} %{public_include_flags} %{internal_include_flags} %{external_include_flags} %{header_deps_flag} %{header_deps_out|concat: $out.d} %{dash_c} $in %{dash_o}$out
 
 rule compile_exe
 %{if header_deps_out}
@@ -37,7 +57,7 @@ rule compile_exe
 %{if ninja_header_deps_style}
   deps = %{ninja_header_deps_style}
 %{endif}
-  command = %{cxx} ${ABI_FLAGS} ${LANG_FLAGS} ${CXXFLAGS} -DBOTAN_IS_BEING_BUILT ${WARN_FLAGS} %{public_include_flags} %{internal_include_flags} %{external_include_flags} %{header_deps_flag} %{header_deps_out|concat: $out.d} %{dash_c} $in %{dash_o}$out
+  command = %{cxx} %{pch_include_for_exe} ${ABI_FLAGS} ${LANG_FLAGS} ${CXXFLAGS} -DBOTAN_IS_BEING_BUILT ${WARN_FLAGS} %{public_include_flags} %{internal_include_flags} %{external_include_flags} %{header_deps_flag} %{header_deps_out|concat: $out.d} %{dash_c} $in %{dash_o}$out
 
 rule compile_example_exe
 %{if header_deps_out}
@@ -195,6 +215,10 @@ build libs: phony %{library_targets} $
 
 build docs: phony %{doc_stamp_file}
 
+%{if enable_pch}
+build pch: phony %{pch_path_for_lib} %{pch_path_for_exe}
+%{endif}
+
 build clean: clean
 
 build distclean: distclean
@@ -211,15 +235,15 @@ build tidy: tidy
 # Build Commands
 
 %{for lib_build_info}
-build %{obj}: compile_lib %{src}
+build %{obj}: compile_lib %{src} | %{pch_target}
 %{endfor}
 
 %{for cli_build_info}
-build %{obj}: compile_exe %{src}
+build %{obj}: compile_exe %{src} | %{pch_target}
 %{endfor}
 
 %{for test_build_info}
-build %{obj}: compile_exe %{src}
+build %{obj}: compile_exe %{src} | %{pch_target}
 %{endfor}
 
 %{for fuzzer_build_info}

src/configs/repo_config.env

@@ -40,3 +40,6 @@ ACVP_SERVER_GIT_URL=https://github.com/usnistgov/ACVP-Server.git
 # Those variables are directly consumed by ccache and sccache respectively
 CCACHE_MAXSIZE="300M"
 SCCACHE_CACHE_SIZE="300M"
+
+# Allow ccache to play nicely with PCH
+CCACHE_SLOPPINESS="pch_defines,time_macros"

src/lib/pch/pch.h

@@ -0,0 +1,34 @@
+/* IWYU pragma: begin_exports */
+
+#include <algorithm>
+#include <array>
+#include <bit>
+/*
+* Note: <chrono> is intentionally omitted here, as even instantiating
+* the templates in it from the PCH is so expensive that it is overall
+* faster to not precompile it and accept the cost in the small number of
+* files which continue to use <chrono>
+*/
+#include <concepts>
+#include <cstddef>
+#include <cstdint>
+#include <cstring>
+#include <exception>
+#include <functional>
+#include <iosfwd>
+#include <limits>
+#include <memory>
+#include <optional>
+#include <ranges>
+#include <span>
+#include <sstream>
+#include <stdexcept>
+#include <string>
+#include <string_view>
+#include <tuple>
+#include <type_traits>
+#include <utility>
+#include <variant>
+#include <vector>
+
+/* IWYU pragma: end_exports */

src/lib/utils/ghash/ghash_cpu/polyval_fn.h

@@ -9,6 +9,10 @@
 
 #include <botan/internal/simd_4x32.h>
 
+#if defined(BOTAN_SIMD_USE_SSSE3)
+   #include <wmmintrin.h>
+#endif
+
 namespace Botan {
 
 // NOLINTBEGIN(portability-simd-intrinsics)

src/lib/utils/simd/simd_2x64/simd_2x64.h

@@ -16,7 +16,6 @@
 // TODO: extend this to support NEON / AltiVec / LSX
 
 #if defined(BOTAN_TARGET_ARCH_SUPPORTS_SSSE3)
-   #include <emmintrin.h>
    #include <tmmintrin.h>
    #define BOTAN_SIMD_USE_SSSE3
 #elif defined(BOTAN_TARGET_ARCH_SUPPORTS_SIMD128)

src/lib/utils/simd/simd_4x32/simd_4x32.h

@@ -15,7 +15,7 @@
 #include <span>
 
 #if defined(BOTAN_TARGET_ARCH_SUPPORTS_SSSE3)
-   #include <immintrin.h>
+   #include <tmmintrin.h>
    #define BOTAN_SIMD_USE_SSSE3
 
 #elif defined(BOTAN_TARGET_ARCH_SUPPORTS_ALTIVEC)

src/lib/utils/simd/simd_hwaes/simd_hwaes.h

@@ -10,6 +10,7 @@
 #include <botan/internal/gfni_utils.h>
 #include <botan/internal/isa_extn.h>
 #include <botan/internal/simd_4x32.h>
+#include <wmmintrin.h>
 
 namespace Botan {