diff --git a/configure.py b/configure.py index bc836f51365..80149dc5deb 100755 --- a/configure.py +++ b/configure.py @@ -250,6 +250,7 @@ def __init__(self, source_paths, options, modules): self.libobj_dir = os.path.join(self.build_dir, 'obj', 'lib') self.cliobj_dir = os.path.join(self.build_dir, 'obj', 'cli') self.testobj_dir = os.path.join(self.build_dir, 'obj', 'test') + self.pch_dir = os.path.join(self.build_dir, 'obj', 'pch') if options.enable_pch else None self.doc_output_dir = os.path.join(self.build_dir, 'docs') self.handbook_output_dir = os.path.join(self.doc_output_dir, 'handbook') @@ -327,6 +328,8 @@ def build_dirs(self): out += [self.fuzzobj_dir, self.fuzzer_output_dir] if self.example_output_dir: out += [self.example_obj_dir, self.example_output_dir] + if self.pch_dir: + out += [self.pch_dir] return out def format_public_include_flags(self, cc): @@ -564,6 +567,8 @@ def add_enable_disable_pair(group, what, default, msg=optparse.SUPPRESS_HELP): default=True, action='store_false', help=optparse.SUPPRESS_HELP) + add_enable_disable_pair(build_group, 'pch', True, 'disable precompiled headers') + add_with_without_pair(build_group, 'valgrind', False, 'use valgrind API') build_group.add_option('--unsafe-fuzzer-mode', action='store_true', default=False, @@ -1325,6 +1330,9 @@ def __init__(self, infofile): 'ninja_header_deps_style': '', 'header_deps_flag': '', 'header_deps_out': '', + 'pch_compile': None, + 'pch_suffix': None, + 'pch_include': None, }) self.add_framework_option = lex.add_framework_option @@ -1373,6 +1381,9 @@ def __init__(self, infofile): self.header_deps_flag = lex.header_deps_flag self.header_deps_out = lex.header_deps_out self.ct_value_barrier = lex.ct_value_barrier + self.pch_compile = lex.pch_compile + self.pch_suffix = lex.pch_suffix + self.pch_include = lex.pch_include def cross_check(self, os_info, arch_info, all_isas): @@ -1419,9 +1430,8 @@ def gen_lib_flags(self, options, variables): """ def flag_builder(): - # We always emit -fPIC or equivalent so that position independent executables - # can be created that link to the static library - yield self.shared_flags + # -fPIC or equiv is emitted via cc_compile_flags so that it's used for all objects + # (this is mostly for the benefit of being able to share the PCH) if options.build_shared_lib: yield self.visibility_build_flags @@ -1429,7 +1439,6 @@ def flag_builder(): if 'debug' in self.lib_flags and options.with_debug_info: yield process_template_string(self.lib_flags['debug'], variables, self.infofile) - return ' '.join(list(flag_builder())) def gen_visibility_attribute(self, options): @@ -1458,6 +1467,12 @@ def ct_value_barrier_type(self, options): return None + def supports_pch(self): + if self.pch_compile is not None: + return self.pch_compile != '' + else: + return False + def mach_abi_link_flags(self, options, debug_mode=None): """ @@ -1561,12 +1576,17 @@ def cc_compile_flags(self, options): if options.cxxflags: # CXXFLAGS is assumed to be the entire set of desired compilation flags # if not the case the user should have used --extra-cxxflags + if self.shared_flags != '': + yield self.shared_flags yield options.cxxflags return if options.with_debug_info: yield self.debug_info_flags + if self.shared_flags != '': + yield self.shared_flags + if not options.no_optimizations: if options.optimize_for_size: if self.size_optimization_flags != '': @@ -2248,6 +2268,10 @@ def test_exe_extra_ldflags(): 'with_doxygen': options.with_doxygen, 'maintainer_mode': options.maintainer_mode, + 'enable_pch': options.enable_pch and cc.supports_pch(), + 'pch_suffix': cc.pch_suffix or '', + 'pch_compile': cc.pch_compile or '', + 'out_dir': normalize_source_path(build_dir), 'build_dir': normalize_source_path(build_paths.build_dir), 'module_info_dir': build_paths.doc_module_info, @@ -2421,6 +2445,29 @@ def test_exe_extra_ldflags(): variables['lib_flags'] = cc.gen_lib_flags(options, variables) + if variables['enable_pch']: + variables['pch_dir'] = build_paths.pch_dir + variables['pch_src_header'] = os.path.join(source_paths.lib_dir, 'pch/pch.h') + variables['pch_header_for_lib'] = os.path.join(build_paths.pch_dir, 'pch_lib.h') + variables['pch_include_for_lib'] = '%s %s' % (cc.pch_include, variables['pch_header_for_lib']) + variables['pch_path_for_lib'] = variables['pch_header_for_lib'] + '.' + cc.pch_suffix + + variables['pch_header_for_exe'] = os.path.join(build_paths.pch_dir, 'pch_exe.h') + variables['pch_include_for_exe'] = '%s %s' % (cc.pch_include, variables['pch_header_for_exe']) + variables['pch_path_for_exe'] = variables['pch_header_for_exe'] + '.' + cc.pch_suffix + + variables['pch_target'] = 'pch' + else: + variables['pch_src_header'] = '' + variables['pch_header_for_lib'] = '' + variables['pch_include_for_lib'] = '' + variables['pch_path_for_lib'] = '' + variables['pch_header_for_exe'] = '' + variables['pch_include_for_exe'] = '' + variables['pch_path_for_exe'] = '' + + variables['pch_target'] = '' + if options.with_pkg_config: variables['botan_pkgconfig'] = os.path.join(build_paths.build_dir, 'botan-%d.pc' % (Version.major())) if options.with_cmake_config: diff --git a/doc/dev_ref/contributing.rst b/doc/dev_ref/contributing.rst index d2172bcd6a9..de955d54150 100644 --- a/doc/dev_ref/contributing.rst +++ b/doc/dev_ref/contributing.rst @@ -154,6 +154,9 @@ If you don't already use it for all your C/C++ development, install ``ccache`` (or on Windows, ``sccache``) right now, and configure a large cache on a fast disk. It allows for very quick rebuilds by caching the compiler output. +For ``ccache``, setting ``CCACHE_SLOPPINESS=pch_defines,time_macros`` allows +using both the compiler cache and precompiled headers, for best compilation speed. + Use ``--enable-sanitizers=`` flag to enable various sanitizer checks. Supported values including "address" and "undefined" for GCC and Clang. GCC also supports "iterator" (checked iterators), and Clang supports "memory" (MSan) and diff --git a/src/build-data/cc/clang.txt b/src/build-data/cc/clang.txt index d89da5deb7c..37b32b0e8f9 100644 --- a/src/build-data/cc/clang.txt +++ b/src/build-data/cc/clang.txt @@ -21,6 +21,10 @@ size_optimization_flags "-Os" add_system_include_dir_option "-isystem" add_sysroot_option "--sysroot=" +pch_suffix "pch" +pch_compile "-Xclang -fno-pch-timestamp -x c++-header" +pch_include "-Xclang -fno-pch-timestamp -include" + default -> address,undefined diff --git a/src/build-data/cc/gcc.txt b/src/build-data/cc/gcc.txt index 6a650589086..1deb0546d39 100644 --- a/src/build-data/cc/gcc.txt +++ b/src/build-data/cc/gcc.txt @@ -7,13 +7,13 @@ minimum_supported_version 11.0 lang_flags "-std=c++20 -D_REENTRANT" # This should only contain flags which are included in GCC 11 -warning_flags "-Wall -Wextra -Wpedantic -Wstrict-aliasing -Wcast-align -Wmissing-declarations -Wpointer-arith -Wcast-qual -Wzero-as-null-pointer-constant -Wnon-virtual-dtor -Wold-style-cast -Wsuggest-override -Wshadow -Wextra-semi" +warning_flags "-Wall -Wextra -Wpedantic -Wstrict-aliasing -Wcast-align -Wmissing-declarations -Wpointer-arith -Wcast-qual -Wzero-as-null-pointer-constant -Wnon-virtual-dtor -Wold-style-cast -Wsuggest-override -Wshadow -Wextra-semi -Winvalid-pch" # Boost headers have 0 as nullptr and non-virtual-dtor issues so we can't werror on them # -Wmaybe-uninitialized is buggy https://gcc.gnu.org/bugzilla/show_bug.cgi?id=105937 # -Wstringop-overread is buggy https://gcc.gnu.org/bugzilla/show_bug.cgi?id=111499 # -Wfree-nonheap-object is buggy https://gcc.gnu.org/bugzilla/show_bug.cgi?id=115016 -werror_flags "-Werror -Wno-error=strict-overflow -Wno-error=zero-as-null-pointer-constant -Wno-error=non-virtual-dtor -Wno-error=maybe-uninitialized -Wno-error=stringop-overread -Wno-error=stringop-overflow -Wno-error=free-nonheap-object -Wno-error=restrict" +werror_flags "-Werror -Wno-error=strict-overflow -Wno-error=zero-as-null-pointer-constant -Wno-error=non-virtual-dtor -Wno-error=maybe-uninitialized -Wno-error=stringop-overread -Wno-error=stringop-overflow -Wno-error=free-nonheap-object -Wno-error=array-bounds -Wno-error=restrict" maintainer_warning_flags "-Wstrict-overflow=5" @@ -30,6 +30,10 @@ stack_protector_flags "-fstack-protector" add_system_include_dir_option "-isystem" add_sysroot_option "--sysroot=" +pch_compile "-x c++-header" +pch_suffix "gch" +pch_include "-include" + default -> iterator,address diff --git a/src/build-data/cc/xcode.txt b/src/build-data/cc/xcode.txt index 55f2211deca..7a61231de1d 100644 --- a/src/build-data/cc/xcode.txt +++ b/src/build-data/cc/xcode.txt @@ -21,6 +21,10 @@ size_optimization_flags "-Os" add_system_include_dir_option "-isystem" add_sysroot_option "--sysroot=" +pch_suffix "pch" +pch_compile "-Xclang -fno-pch-timestamp -x c++-header" +pch_include "-Xclang -fno-pch-timestamp -include" + default -> address,undefined diff --git a/src/build-data/makefile.in b/src/build-data/makefile.in index 2e673179498..9d8ec5a65b8 100644 --- a/src/build-data/makefile.in +++ b/src/build-data/makefile.in @@ -51,6 +51,22 @@ docs: %{doc_stamp_file} .PHONY: all cli libs tests check docs clean distclean install fmt tidy %{endif} +%{if enable_pch} + +%{pch_header_for_lib}: %{pch_src_header} + "$(PYTHON_EXE)" -c "import shutil, sys; shutil.copyfile(sys.argv[1], sys.argv[2])" "$<" "$@" + +%{pch_header_for_exe}: %{pch_src_header} + "$(PYTHON_EXE)" -c "import shutil, sys; shutil.copyfile(sys.argv[1], sys.argv[2])" "$<" "$@" + +%{pch_path_for_lib}: %{pch_header_for_lib} + $(CXX) $(LIB_FLAGS) $(BUILD_FLAGS) -DBOTAN_IS_BEING_BUILT %{public_include_flags} %{internal_include_flags} %{external_include_flags} %{pch_compile} %{dash_c} %{pch_header_for_lib} %{dash_o}$@ + +%{pch_path_for_exe}: %{pch_header_for_exe} + $(CXX) $(BUILD_FLAGS) -DBOTAN_IS_BEING_BUILT %{public_include_flags} %{internal_include_flags} %{external_include_flags} %{pch_compile} %{dash_c} %{pch_header_for_exe} %{dash_o}$@ + +%{endif} + %{doc_stamp_file}: %{doc_dir}/*.rst %{doc_dir}/api_ref/*.rst %{doc_dir}/dev_ref/*.rst "$(PYTHON_EXE)" "$(SCRIPTS_DIR)/build_docs.py" --build-dir="%{build_dir}" @@ -150,18 +166,18 @@ ct_selftest: %{out_dir}/botan_ct_selftest # Build Commands %{for lib_build_info} -%{obj}: %{src} - $(CXX) $(LIB_FLAGS) $(BUILD_FLAGS) -DBOTAN_IS_BEING_BUILT %{public_include_flags} %{internal_include_flags} %{external_include_flags} %{dash_c} %{src} %{dash_o}$@ +%{obj}: %{pch_path_for_lib} %{src} + $(CXX) %{pch_include_for_lib} $(LIB_FLAGS) $(BUILD_FLAGS) -DBOTAN_IS_BEING_BUILT %{public_include_flags} %{internal_include_flags} %{external_include_flags} %{dash_c} %{src} %{dash_o}$@ %{endfor} %{for cli_build_info} -%{obj}: %{src} - $(CXX) $(BUILD_FLAGS) -DBOTAN_IS_BEING_BUILT %{public_include_flags} %{internal_include_flags} %{external_include_flags} %{dash_c} %{src} %{dash_o}$@ +%{obj}: %{pch_path_for_exe} %{src} + $(CXX) %{pch_include_for_exe} $(BUILD_FLAGS) -DBOTAN_IS_BEING_BUILT %{public_include_flags} %{internal_include_flags} %{external_include_flags} %{dash_c} %{src} %{dash_o}$@ %{endfor} %{for test_build_info} -%{obj}: %{src} - $(CXX) $(BUILD_FLAGS) -DBOTAN_IS_BEING_BUILT %{public_include_flags} %{internal_include_flags} %{external_include_flags} %{dash_c} %{src} %{dash_o}$@ +%{obj}: %{pch_path_for_exe} %{src} + $(CXX) %{pch_include_for_exe} $(BUILD_FLAGS) -DBOTAN_IS_BEING_BUILT %{public_include_flags} %{internal_include_flags} %{external_include_flags} %{dash_c} %{src} %{dash_o}$@ %{endfor} %{for fuzzer_build_info} diff --git a/src/build-data/ninja.in b/src/build-data/ninja.in index c06050fafa4..6250205af83 100644 --- a/src/build-data/ninja.in +++ b/src/build-data/ninja.in @@ -21,6 +21,26 @@ EXE_LINKS_TO = %{link_to_botan} ${LIB_LINKS_TO} %{extra_libs} SCRIPTS_DIR = %{scripts_dir} INSTALLED_LIB_DIR = %{libdir} +%{if enable_pch} +rule copy_pch_header + command = "${PYTHON_EXE}" -c "import shutil, sys; shutil.copyfile(sys.argv[1], sys.argv[2])" "$in" "$out" + +rule compile_pch_lib + command = %{cxx} %{lib_flags} ${ABI_FLAGS} ${LANG_FLAGS} ${CXXFLAGS} -DBOTAN_IS_BEING_BUILT ${WARN_FLAGS} %{public_include_flags} %{internal_include_flags} %{external_include_flags} %{pch_compile} %{dash_c} $in %{dash_o}$out + +rule compile_pch_exe + command = %{cxx} ${ABI_FLAGS} ${LANG_FLAGS} ${CXXFLAGS} -DBOTAN_IS_BEING_BUILT ${WARN_FLAGS} %{public_include_flags} %{internal_include_flags} %{external_include_flags} %{pch_compile} %{dash_c} $in %{dash_o}$out + +build %{pch_header_for_lib}: copy_pch_header %{pch_src_header} + +build %{pch_header_for_exe}: copy_pch_header %{pch_src_header} + +build %{pch_path_for_lib}: compile_pch_lib %{pch_header_for_lib} + +build %{pch_path_for_exe}: compile_pch_exe %{pch_header_for_exe} + +%{endif} + rule compile_lib %{if header_deps_out} depfile = $out.d @@ -28,7 +48,7 @@ rule compile_lib %{if ninja_header_deps_style} deps = %{ninja_header_deps_style} %{endif} - command = %{cxx} %{lib_flags} ${ABI_FLAGS} ${LANG_FLAGS} ${CXXFLAGS} -DBOTAN_IS_BEING_BUILT ${WARN_FLAGS} %{public_include_flags} %{internal_include_flags} %{external_include_flags} %{header_deps_flag} %{header_deps_out|concat: $out.d} %{dash_c} $in %{dash_o}$out + command = %{cxx} %{pch_include_for_lib} %{lib_flags} ${ABI_FLAGS} ${LANG_FLAGS} ${CXXFLAGS} -DBOTAN_IS_BEING_BUILT ${WARN_FLAGS} %{public_include_flags} %{internal_include_flags} %{external_include_flags} %{header_deps_flag} %{header_deps_out|concat: $out.d} %{dash_c} $in %{dash_o}$out rule compile_exe %{if header_deps_out} @@ -37,7 +57,7 @@ rule compile_exe %{if ninja_header_deps_style} deps = %{ninja_header_deps_style} %{endif} - command = %{cxx} ${ABI_FLAGS} ${LANG_FLAGS} ${CXXFLAGS} -DBOTAN_IS_BEING_BUILT ${WARN_FLAGS} %{public_include_flags} %{internal_include_flags} %{external_include_flags} %{header_deps_flag} %{header_deps_out|concat: $out.d} %{dash_c} $in %{dash_o}$out + command = %{cxx} %{pch_include_for_exe} ${ABI_FLAGS} ${LANG_FLAGS} ${CXXFLAGS} -DBOTAN_IS_BEING_BUILT ${WARN_FLAGS} %{public_include_flags} %{internal_include_flags} %{external_include_flags} %{header_deps_flag} %{header_deps_out|concat: $out.d} %{dash_c} $in %{dash_o}$out rule compile_example_exe %{if header_deps_out} @@ -195,6 +215,10 @@ build libs: phony %{library_targets} $ build docs: phony %{doc_stamp_file} +%{if enable_pch} +build pch: phony %{pch_path_for_lib} %{pch_path_for_exe} +%{endif} + build clean: clean build distclean: distclean @@ -211,15 +235,15 @@ build tidy: tidy # Build Commands %{for lib_build_info} -build %{obj}: compile_lib %{src} +build %{obj}: compile_lib %{src} | %{pch_target} %{endfor} %{for cli_build_info} -build %{obj}: compile_exe %{src} +build %{obj}: compile_exe %{src} | %{pch_target} %{endfor} %{for test_build_info} -build %{obj}: compile_exe %{src} +build %{obj}: compile_exe %{src} | %{pch_target} %{endfor} %{for fuzzer_build_info} diff --git a/src/configs/repo_config.env b/src/configs/repo_config.env index bb1df5d03b4..02bc60fe5e6 100644 --- a/src/configs/repo_config.env +++ b/src/configs/repo_config.env @@ -40,3 +40,6 @@ ACVP_SERVER_GIT_URL=https://github.com/usnistgov/ACVP-Server.git # Those variables are directly consumed by ccache and sccache respectively CCACHE_MAXSIZE="300M" SCCACHE_CACHE_SIZE="300M" + +# Allow ccache to play nicely with PCH +CCACHE_SLOPPINESS="pch_defines,time_macros" diff --git a/src/lib/pch/pch.h b/src/lib/pch/pch.h new file mode 100644 index 00000000000..c383c7898d5 --- /dev/null +++ b/src/lib/pch/pch.h @@ -0,0 +1,34 @@ +/* IWYU pragma: begin_exports */ + +#include +#include +#include +/* +* Note: is intentionally omitted here, as even instantiating +* the templates in it from the PCH is so expensive that it is overall +* faster to not precompile it and accept the cost in the small number of +* files which continue to use +*/ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* IWYU pragma: end_exports */ diff --git a/src/lib/utils/ghash/ghash_cpu/polyval_fn.h b/src/lib/utils/ghash/ghash_cpu/polyval_fn.h index f38a431e755..037afc0143d 100644 --- a/src/lib/utils/ghash/ghash_cpu/polyval_fn.h +++ b/src/lib/utils/ghash/ghash_cpu/polyval_fn.h @@ -9,6 +9,10 @@ #include +#if defined(BOTAN_SIMD_USE_SSSE3) + #include +#endif + namespace Botan { // NOLINTBEGIN(portability-simd-intrinsics) diff --git a/src/lib/utils/simd/simd_2x64/simd_2x64.h b/src/lib/utils/simd/simd_2x64/simd_2x64.h index 6d3bc63c4a8..02f28c9150f 100644 --- a/src/lib/utils/simd/simd_2x64/simd_2x64.h +++ b/src/lib/utils/simd/simd_2x64/simd_2x64.h @@ -16,7 +16,6 @@ // TODO: extend this to support NEON / AltiVec / LSX #if defined(BOTAN_TARGET_ARCH_SUPPORTS_SSSE3) - #include #include #define BOTAN_SIMD_USE_SSSE3 #elif defined(BOTAN_TARGET_ARCH_SUPPORTS_SIMD128) diff --git a/src/lib/utils/simd/simd_4x32/simd_4x32.h b/src/lib/utils/simd/simd_4x32/simd_4x32.h index d774145b58b..0d8db3ab84f 100644 --- a/src/lib/utils/simd/simd_4x32/simd_4x32.h +++ b/src/lib/utils/simd/simd_4x32/simd_4x32.h @@ -15,7 +15,7 @@ #include #if defined(BOTAN_TARGET_ARCH_SUPPORTS_SSSE3) - #include + #include #define BOTAN_SIMD_USE_SSSE3 #elif defined(BOTAN_TARGET_ARCH_SUPPORTS_ALTIVEC) diff --git a/src/lib/utils/simd/simd_hwaes/simd_hwaes.h b/src/lib/utils/simd/simd_hwaes/simd_hwaes.h index 7640dc1ab90..963de257d40 100644 --- a/src/lib/utils/simd/simd_hwaes/simd_hwaes.h +++ b/src/lib/utils/simd/simd_hwaes/simd_hwaes.h @@ -10,6 +10,7 @@ #include #include #include +#include namespace Botan { diff --git a/src/scripts/cleanup.py b/src/scripts/cleanup.py index d020576a4f3..bff75f94b52 100755 --- a/src/scripts/cleanup.py +++ b/src/scripts/cleanup.py @@ -109,6 +109,14 @@ def main(args=None): remove_file(build_config['doc_stamp_file']) + pch_suffixes = ['pch', 'gch'] + + if 'pch_dir' in build_config: + for f in os.listdir(build_config['pch_dir']): + for suffix in pch_suffixes: + if f.endswith(suffix): + remove_file(os.path.join(build_config['pch_dir'], f)) + remove_file(build_config['cli_exe']) remove_file(build_config['test_exe'])